[llvm] 5515188 - [AMDGPU] Select S_CSELECT_B32/64 for uniform extensions from i1 (#204238)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 03:24:03 PDT 2026
Author: Jay Foad
Date: 2026-06-30T11:23:34+01:00
New Revision: 5515188560107ef07fe1085d6779915ed47b32c0
URL: https://github.com/llvm/llvm-project/commit/5515188560107ef07fe1085d6779915ed47b32c0
DIFF: https://github.com/llvm/llvm-project/commit/5515188560107ef07fe1085d6779915ed47b32c0.diff
LOG: [AMDGPU] Select S_CSELECT_B32/64 for uniform extensions from i1 (#204238)
Fixes #59869
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/lib/Target/AMDGPU/SIInstructions.td
llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
llvm/test/CodeGen/AMDGPU/add.ll
llvm/test/CodeGen/AMDGPU/add_i1.ll
llvm/test/CodeGen/AMDGPU/addsub64_carry.ll
llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
llvm/test/CodeGen/AMDGPU/anyext.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
llvm/test/CodeGen/AMDGPU/bug-cselect-b64.ll
llvm/test/CodeGen/AMDGPU/carryout-selection.ll
llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll
llvm/test/CodeGen/AMDGPU/ctpop.ll
llvm/test/CodeGen/AMDGPU/ctpop16.ll
llvm/test/CodeGen/AMDGPU/ctpop64.ll
llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll
llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
llvm/test/CodeGen/AMDGPU/exec-mask-opt-cannot-create-empty-or-backward-segment.ll
llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
llvm/test/CodeGen/AMDGPU/fp-classify.ll
llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
llvm/test/CodeGen/AMDGPU/fptrunc.ll
llvm/test/CodeGen/AMDGPU/freeze.ll
llvm/test/CodeGen/AMDGPU/frem.ll
llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll
llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll
llvm/test/CodeGen/AMDGPU/issue176578.ll
llvm/test/CodeGen/AMDGPU/large-phi-search.ll
llvm/test/CodeGen/AMDGPU/literal64.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
llvm/test/CodeGen/AMDGPU/long-branch-reserve-register.ll
llvm/test/CodeGen/AMDGPU/mad-combine.ll
llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
llvm/test/CodeGen/AMDGPU/min.ll
llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
llvm/test/CodeGen/AMDGPU/mul.ll
llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
llvm/test/CodeGen/AMDGPU/or.ll
llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll
llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
llvm/test/CodeGen/AMDGPU/saddo.ll
llvm/test/CodeGen/AMDGPU/sdiv64.ll
llvm/test/CodeGen/AMDGPU/select-undef.ll
llvm/test/CodeGen/AMDGPU/selectcc-opt.ll
llvm/test/CodeGen/AMDGPU/setcc-opt.ll
llvm/test/CodeGen/AMDGPU/setcc.ll
llvm/test/CodeGen/AMDGPU/setcc64.ll
llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
llvm/test/CodeGen/AMDGPU/sgpr-copy.ll
llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
llvm/test/CodeGen/AMDGPU/si-unify-exit-return-unreachable.ll
llvm/test/CodeGen/AMDGPU/sign_extend.ll
llvm/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
llvm/test/CodeGen/AMDGPU/sink-addr-memory-intrinsics.ll
llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
llvm/test/CodeGen/AMDGPU/srem.ll
llvm/test/CodeGen/AMDGPU/srem64.ll
llvm/test/CodeGen/AMDGPU/ssubo.ll
llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
llvm/test/CodeGen/AMDGPU/sub-zext-cc-zext-cc.ll
llvm/test/CodeGen/AMDGPU/sub_i1.ll
llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
llvm/test/CodeGen/AMDGPU/uaddo.ll
llvm/test/CodeGen/AMDGPU/udiv64.ll
llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
llvm/test/CodeGen/AMDGPU/urem64.ll
llvm/test/CodeGen/AMDGPU/usubo.ll
llvm/test/CodeGen/AMDGPU/v_cmp_gfx11.ll
llvm/test/CodeGen/AMDGPU/wave32.ll
llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
llvm/test/CodeGen/AMDGPU/workitem-intrinsic-opts.ll
llvm/test/CodeGen/AMDGPU/wqm.ll
llvm/test/CodeGen/AMDGPU/xor.ll
llvm/test/CodeGen/AMDGPU/zero_extend.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 48953cee43956..48a85fdc01ea6 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -8697,14 +8697,30 @@ void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
Register CondReg = Cond.getReg();
bool IsSCC = (CondReg == AMDGPU::SCC);
- // If this is a trivial select where the condition is effectively not SCC
- // (CondReg is a source of copy to SCC), then the select is semantically
- // equivalent to copying CondReg. Hence, there is no need to create
- // V_CNDMASK, we can just use that and bail out.
+ // Remove S_CSELECT instructions that we previously inserted to feed the SCC
+ // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
+ // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
+ // output directly into the V_CNDMASK.
if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
(Src1.getImm() == 0)) {
- MRI.replaceRegWith(Dest.getReg(), CondReg);
- return;
+ for (MachineOperand &UseMO :
+ make_early_inc_range(MRI.use_nodbg_operands(Dest.getReg()))) {
+ MachineInstr &UseMI = *UseMO.getParent();
+ switch (UseMI.getOpcode()) {
+ case AMDGPU::V_CNDMASK_B16_fake16_e32:
+ case AMDGPU::V_CNDMASK_B16_fake16_e64:
+ case AMDGPU::V_CNDMASK_B16_t16_e32:
+ case AMDGPU::V_CNDMASK_B16_t16_e64:
+ case AMDGPU::V_CNDMASK_B32_e32:
+ case AMDGPU::V_CNDMASK_B32_e64:
+ case AMDGPU::V_CNDMASK_B64_PSEUDO:
+ if (UseMO.isImplicit() ||
+ &UseMO == getNamedOperand(UseMI, AMDGPU::OpName::src2))
+ UseMO.setReg(CondReg);
+ }
+ }
+ if (MRI.use_nodbg_empty(Dest.getReg()))
+ return;
}
Register NewCondReg = CondReg;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 86f78762d36eb..15bc6704e076b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2795,6 +2795,23 @@ class Ext32Pat <SDNode ext> : GCNPat <
def : Ext32Pat <zext>;
def : Ext32Pat <anyext>;
+// Uniform extensions from i1
+
+foreach ext = [sext, zext, anyext] in {
+ defvar extone = !switch(ext, sext: -1, 1);
+
+ foreach vt = [i16, i32] in
+ def : GCNPat <
+ (vt (UniformUnaryFrag<ext> SCC)),
+ (S_CSELECT_B32 (vt extone), (vt 0))
+ >;
+
+ def : GCNPat <
+ (i64 (UniformUnaryFrag<ext> SCC)),
+ (S_CSELECT_B64 (i64 extone), (i64 0))
+ >;
+}
+
// The multiplication scales from [0,1) to the unsigned integer range,
// rounding down a bit to avoid unwanted overflow.
def : GCNPat <
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index 7d0d6eb3f7c91..241050c7bb926 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -14483,26 +14483,32 @@ define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB193_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB193_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a3, v1
; GFX90A-NEXT: v_accvgpr_write_b32 a2, v0
-; GFX90A-NEXT: s_cbranch_execz .LBB193_3
-; GFX90A-NEXT: s_branch .LBB193_4
+; GFX90A-NEXT: s_branch .LBB193_3
; GFX90A-NEXT: .LBB193_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr2_agpr3
-; GFX90A-NEXT: .LBB193_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB193_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB193_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v0, s4
@@ -14511,7 +14517,7 @@ define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_store_dword a0, v0, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword a1, v0, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB193_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB193_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(2)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[2:3]
@@ -14526,32 +14532,38 @@ define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB193_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB193_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a3, v1
; GFX950-NEXT: v_accvgpr_write_b32 a2, v0
-; GFX950-NEXT: s_cbranch_execz .LBB193_3
-; GFX950-NEXT: s_branch .LBB193_4
+; GFX950-NEXT: s_branch .LBB193_3
; GFX950-NEXT: .LBB193_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr2_agpr3
-; GFX950-NEXT: .LBB193_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB193_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB193_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 a[2:3], off, s0
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: scratch_store_dwordx2 off, a[0:1], s0
-; GFX950-NEXT: .LBB193_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB193_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(1)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[2:3]
@@ -14573,21 +14585,27 @@ define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB194_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB194_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_swap_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB194_3
-; GFX90A-NEXT: s_branch .LBB194_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB194_3
; GFX90A-NEXT: .LBB194_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB194_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB194_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB194_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -14596,7 +14614,7 @@ define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB194_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB194_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -14610,27 +14628,33 @@ define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB194_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB194_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_swap_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB194_3
-; GFX950-NEXT: s_branch .LBB194_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB194_3
; GFX950-NEXT: .LBB194_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: .LBB194_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB194_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB194_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB194_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB194_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[2:3]
@@ -14654,23 +14678,29 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB195_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB195_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB195_3
-; GFX90A-NEXT: s_branch .LBB195_4
+; GFX90A-NEXT: s_branch .LBB195_3
; GFX90A-NEXT: .LBB195_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB195_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB195_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB195_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -14684,7 +14714,7 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v1, vcc
; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB195_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB195_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -14701,23 +14731,29 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB195_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB195_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB195_3
-; GFX950-NEXT: s_branch .LBB195_4
+; GFX950-NEXT: s_branch .LBB195_3
; GFX950-NEXT: .LBB195_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB195_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB195_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB195_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -14726,7 +14762,7 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, v[0:1]
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB195_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB195_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -14747,21 +14783,27 @@ define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB196_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB196_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB196_3
-; GFX90A-NEXT: s_branch .LBB196_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB196_3
; GFX90A-NEXT: .LBB196_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB196_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB196_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB196_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -14772,7 +14814,7 @@ define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB196_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB196_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -14786,28 +14828,34 @@ define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB196_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB196_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB196_3
-; GFX950-NEXT: s_branch .LBB196_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB196_3
; GFX950-NEXT: .LBB196_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: .LBB196_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB196_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB196_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, v[0:1]
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB196_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB196_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[2:3]
@@ -14831,23 +14879,29 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB197_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB197_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB197_3
-; GFX90A-NEXT: s_branch .LBB197_4
+; GFX90A-NEXT: s_branch .LBB197_3
; GFX90A-NEXT: .LBB197_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB197_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB197_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB197_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -14861,7 +14915,7 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v4, v1, vcc
; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB197_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB197_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -14878,23 +14932,29 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB197_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB197_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB197_3
-; GFX950-NEXT: s_branch .LBB197_4
+; GFX950-NEXT: s_branch .LBB197_3
; GFX950-NEXT: .LBB197_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB197_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB197_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB197_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -14905,7 +14965,7 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB197_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB197_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -14926,21 +14986,27 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB198_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB198_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB198_3
-; GFX90A-NEXT: s_branch .LBB198_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB198_3
; GFX90A-NEXT: .LBB198_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB198_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB198_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB198_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -14951,7 +15017,7 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB198_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB198_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -14965,21 +15031,27 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB198_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB198_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB198_3
-; GFX950-NEXT: s_branch .LBB198_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB198_3
; GFX950-NEXT: .LBB198_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB198_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB198_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB198_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -14988,7 +15060,7 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB198_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB198_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -15012,23 +15084,29 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB199_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB199_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB199_3
-; GFX90A-NEXT: s_branch .LBB199_4
+; GFX90A-NEXT: s_branch .LBB199_3
; GFX90A-NEXT: .LBB199_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB199_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB199_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB199_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -15042,7 +15120,7 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_and_b32_e32 v1, v4, v1
; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB199_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB199_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -15059,23 +15137,29 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB199_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB199_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB199_3
-; GFX950-NEXT: s_branch .LBB199_4
+; GFX950-NEXT: s_branch .LBB199_3
; GFX950-NEXT: .LBB199_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB199_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB199_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB199_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -15085,7 +15169,7 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_and_b32_e32 v0, v2, v0
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB199_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB199_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -15106,21 +15190,27 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB200_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB200_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB200_3
-; GFX90A-NEXT: s_branch .LBB200_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB200_3
; GFX90A-NEXT: .LBB200_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB200_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB200_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB200_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -15131,7 +15221,7 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_and_b32_e32 v0, v2, v0
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB200_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB200_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -15145,21 +15235,27 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB200_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB200_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB200_3
-; GFX950-NEXT: s_branch .LBB200_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB200_3
; GFX950-NEXT: .LBB200_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB200_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB200_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB200_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -15167,7 +15263,7 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_and_b32_e32 v3, v1, v3
; GFX950-NEXT: v_and_b32_e32 v2, v0, v2
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB200_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB200_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -15191,11 +15287,11 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB201_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB201_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15255,11 +15351,11 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB201_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB201_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15319,12 +15415,12 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB202_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB202_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15377,12 +15473,12 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB202_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB202_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15441,23 +15537,29 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB203_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB203_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB203_3
-; GFX90A-NEXT: s_branch .LBB203_4
+; GFX90A-NEXT: s_branch .LBB203_3
; GFX90A-NEXT: .LBB203_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB203_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB203_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB203_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -15471,7 +15573,7 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_or_b32_e32 v1, v4, v1
; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB203_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB203_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -15488,23 +15590,29 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB203_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB203_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB203_3
-; GFX950-NEXT: s_branch .LBB203_4
+; GFX950-NEXT: s_branch .LBB203_3
; GFX950-NEXT: .LBB203_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB203_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB203_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB203_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -15514,7 +15622,7 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_or_b32_e32 v0, v2, v0
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB203_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB203_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -15535,21 +15643,27 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB204_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB204_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB204_3
-; GFX90A-NEXT: s_branch .LBB204_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB204_3
; GFX90A-NEXT: .LBB204_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB204_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB204_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB204_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -15560,7 +15674,7 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_or_b32_e32 v0, v2, v0
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB204_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB204_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -15574,21 +15688,27 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB204_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB204_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB204_3
-; GFX950-NEXT: s_branch .LBB204_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB204_3
; GFX950-NEXT: .LBB204_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB204_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB204_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB204_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -15596,7 +15716,7 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_or_b32_e32 v3, v1, v3
; GFX950-NEXT: v_or_b32_e32 v2, v0, v2
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB204_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB204_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -15620,23 +15740,29 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB205_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB205_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB205_3
-; GFX90A-NEXT: s_branch .LBB205_4
+; GFX90A-NEXT: s_branch .LBB205_3
; GFX90A-NEXT: .LBB205_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB205_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB205_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB205_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -15650,7 +15776,7 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_xor_b32_e32 v1, v4, v1
; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB205_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB205_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -15667,23 +15793,29 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB205_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB205_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB205_3
-; GFX950-NEXT: s_branch .LBB205_4
+; GFX950-NEXT: s_branch .LBB205_3
; GFX950-NEXT: .LBB205_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB205_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB205_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB205_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -15693,7 +15825,7 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_xor_b32_e32 v0, v2, v0
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB205_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB205_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -15714,21 +15846,27 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB206_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB206_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB206_3
-; GFX90A-NEXT: s_branch .LBB206_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB206_3
; GFX90A-NEXT: .LBB206_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB206_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB206_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB206_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -15739,7 +15877,7 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_xor_b32_e32 v0, v2, v0
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB206_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB206_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -15753,21 +15891,27 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB206_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB206_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB206_3
-; GFX950-NEXT: s_branch .LBB206_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB206_3
; GFX950-NEXT: .LBB206_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB206_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB206_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB206_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -15775,7 +15919,7 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_xor_b32_e32 v3, v1, v3
; GFX950-NEXT: v_xor_b32_e32 v2, v0, v2
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB206_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB206_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -15799,23 +15943,29 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB207_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB207_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB207_3
-; GFX90A-NEXT: s_branch .LBB207_4
+; GFX90A-NEXT: s_branch .LBB207_3
; GFX90A-NEXT: .LBB207_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB207_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB207_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB207_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -15830,7 +15980,7 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB207_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB207_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -15847,23 +15997,29 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB207_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB207_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB207_3
-; GFX950-NEXT: s_branch .LBB207_4
+; GFX950-NEXT: s_branch .LBB207_3
; GFX950-NEXT: .LBB207_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB207_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB207_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB207_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -15874,7 +16030,7 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB207_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB207_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -15895,21 +16051,27 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB208_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB208_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB208_3
-; GFX90A-NEXT: s_branch .LBB208_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB208_3
; GFX90A-NEXT: .LBB208_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB208_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB208_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB208_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -15921,7 +16083,7 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB208_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB208_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -15935,21 +16097,27 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB208_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB208_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB208_3
-; GFX950-NEXT: s_branch .LBB208_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB208_3
; GFX950-NEXT: .LBB208_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB208_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB208_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB208_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -15959,7 +16127,7 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB208_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB208_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -15983,23 +16151,29 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB209_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB209_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB209_3
-; GFX90A-NEXT: s_branch .LBB209_4
+; GFX90A-NEXT: s_branch .LBB209_3
; GFX90A-NEXT: .LBB209_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB209_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB209_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB209_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16014,7 +16188,7 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB209_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB209_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -16031,23 +16205,29 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB209_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB209_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB209_3
-; GFX950-NEXT: s_branch .LBB209_4
+; GFX950-NEXT: s_branch .LBB209_3
; GFX950-NEXT: .LBB209_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB209_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB209_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB209_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -16058,7 +16238,7 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB209_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB209_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -16079,21 +16259,27 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB210_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB210_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB210_3
-; GFX90A-NEXT: s_branch .LBB210_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB210_3
; GFX90A-NEXT: .LBB210_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB210_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB210_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB210_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16105,7 +16291,7 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB210_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB210_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -16119,21 +16305,27 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB210_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB210_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB210_3
-; GFX950-NEXT: s_branch .LBB210_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB210_3
; GFX950-NEXT: .LBB210_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB210_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB210_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB210_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -16143,7 +16335,7 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB210_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB210_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -16167,23 +16359,29 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB211_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB211_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB211_3
-; GFX90A-NEXT: s_branch .LBB211_4
+; GFX90A-NEXT: s_branch .LBB211_3
; GFX90A-NEXT: .LBB211_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB211_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB211_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB211_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16198,7 +16396,7 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB211_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB211_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -16215,23 +16413,29 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB211_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB211_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB211_3
-; GFX950-NEXT: s_branch .LBB211_4
+; GFX950-NEXT: s_branch .LBB211_3
; GFX950-NEXT: .LBB211_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB211_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB211_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB211_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -16242,7 +16446,7 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB211_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB211_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -16263,21 +16467,27 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB212_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB212_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB212_3
-; GFX90A-NEXT: s_branch .LBB212_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB212_3
; GFX90A-NEXT: .LBB212_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB212_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB212_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB212_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16289,7 +16499,7 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB212_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB212_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -16303,21 +16513,27 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB212_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB212_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB212_3
-; GFX950-NEXT: s_branch .LBB212_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB212_3
; GFX950-NEXT: .LBB212_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB212_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB212_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB212_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -16327,7 +16543,7 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB212_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB212_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -16351,23 +16567,29 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB213_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB213_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB213_3
-; GFX90A-NEXT: s_branch .LBB213_4
+; GFX90A-NEXT: s_branch .LBB213_3
; GFX90A-NEXT: .LBB213_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB213_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB213_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB213_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16382,7 +16604,7 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB213_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB213_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -16399,23 +16621,29 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB213_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB213_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB213_3
-; GFX950-NEXT: s_branch .LBB213_4
+; GFX950-NEXT: s_branch .LBB213_3
; GFX950-NEXT: .LBB213_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB213_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB213_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB213_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -16426,7 +16654,7 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB213_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB213_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -16447,21 +16675,27 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB214_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB214_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB214_3
-; GFX90A-NEXT: s_branch .LBB214_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB214_3
; GFX90A-NEXT: .LBB214_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB214_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB214_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB214_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16473,7 +16707,7 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB214_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB214_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -16487,21 +16721,27 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB214_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB214_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB214_3
-; GFX950-NEXT: s_branch .LBB214_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB214_3
; GFX950-NEXT: .LBB214_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB214_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB214_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB214_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -16511,7 +16751,7 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB214_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB214_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -16535,23 +16775,29 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB215_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB215_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB215_3
-; GFX90A-NEXT: s_branch .LBB215_4
+; GFX90A-NEXT: s_branch .LBB215_3
; GFX90A-NEXT: .LBB215_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB215_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB215_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB215_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16568,7 +16814,7 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB215_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB215_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -16585,23 +16831,29 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB215_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB215_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB215_3
-; GFX950-NEXT: s_branch .LBB215_4
+; GFX950-NEXT: s_branch .LBB215_3
; GFX950-NEXT: .LBB215_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB215_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB215_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB215_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -16613,7 +16865,7 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB215_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB215_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -16634,21 +16886,27 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB216_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB216_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB216_3
-; GFX90A-NEXT: s_branch .LBB216_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB216_3
; GFX90A-NEXT: .LBB216_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX90A-NEXT: .LBB216_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB216_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB216_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16662,7 +16920,7 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc
; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB216_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB216_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[0:1]
@@ -16676,21 +16934,27 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB216_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB216_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB216_3
-; GFX950-NEXT: s_branch .LBB216_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB216_3
; GFX950-NEXT: .LBB216_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB216_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB216_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB216_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -16701,7 +16965,7 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB216_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB216_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -16725,23 +16989,29 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB217_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB217_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB217_3
-; GFX90A-NEXT: s_branch .LBB217_4
+; GFX90A-NEXT: s_branch .LBB217_3
; GFX90A-NEXT: .LBB217_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB217_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB217_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB217_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16759,7 +17029,7 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB217_4: ; %atomicrmw.phi
+; GFX90A-NEXT: .LBB217_5: ; %atomicrmw.phi
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -16776,23 +17046,29 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB217_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB217_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB217_3
-; GFX950-NEXT: s_branch .LBB217_4
+; GFX950-NEXT: s_branch .LBB217_3
; GFX950-NEXT: .LBB217_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB217_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB217_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB217_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
@@ -16807,7 +17083,7 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s2
-; GFX950-NEXT: .LBB217_4: ; %atomicrmw.phi
+; GFX950-NEXT: .LBB217_5: ; %atomicrmw.phi
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -16828,21 +17104,27 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB218_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB218_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB218_3
-; GFX90A-NEXT: s_branch .LBB218_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB218_3
; GFX90A-NEXT: .LBB218_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB218_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB218_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB218_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v4, s4
@@ -16857,7 +17139,7 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB218_4: ; %atomicrmw.phi
+; GFX90A-NEXT: .LBB218_5: ; %atomicrmw.phi
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -16871,21 +17153,27 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB218_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB218_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB218_3
-; GFX950-NEXT: s_branch .LBB218_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB218_3
; GFX950-NEXT: .LBB218_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB218_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB218_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB218_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s2
@@ -16899,7 +17187,7 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s2
-; GFX950-NEXT: .LBB218_4: ; %atomicrmw.phi
+; GFX950-NEXT: .LBB218_5: ; %atomicrmw.phi
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -16923,11 +17211,11 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB219_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB219_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -16989,11 +17277,11 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB219_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB219_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17059,12 +17347,12 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB220_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB220_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17119,12 +17407,12 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB220_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB220_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17189,11 +17477,11 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB221_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB221_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17253,11 +17541,11 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB221_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB221_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17321,12 +17609,12 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB222_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB222_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17379,12 +17667,12 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB222_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB222_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17448,32 +17736,39 @@ define void @flat_atomic_fadd_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB223_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB223_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB223_4
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB223_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v1
-; GFX90A-NEXT: s_cbranch_execz .LBB223_5
-; GFX90A-NEXT: s_branch .LBB223_6
+; GFX90A-NEXT: s_branch .LBB223_5
; GFX90A-NEXT: .LBB223_3:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0
-; GFX90A-NEXT: s_branch .LBB223_7
+; GFX90A-NEXT: s_branch .LBB223_8
; GFX90A-NEXT: .LBB223_4:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0
-; GFX90A-NEXT: .LBB223_5: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB223_5: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB223_7
+; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s6, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
@@ -17482,16 +17777,21 @@ define void @flat_atomic_fadd_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_add_f32_e32 v3, v2, v0
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB223_6: ; %Flow1
-; GFX90A-NEXT: s_cbranch_execnz .LBB223_8
-; GFX90A-NEXT: .LBB223_7: ; %atomicrmw.shared
+; GFX90A-NEXT: .LBB223_7: ; %Flow1
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: .LBB223_8: ; %Flow2
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB223_10
+; GFX90A-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: ds_add_rtn_f32 v0, v1, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX90A-NEXT: .LBB223_8: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB223_10: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
; GFX90A-NEXT: ;;#ASMEND
@@ -17530,29 +17830,36 @@ define void @flat_atomic_fadd_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB224_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB224_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB224_4
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB224_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc
-; GFX90A-NEXT: s_cbranch_execz .LBB224_5
-; GFX90A-NEXT: s_branch .LBB224_6
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB224_5
; GFX90A-NEXT: .LBB224_3:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_branch .LBB224_7
+; GFX90A-NEXT: s_branch .LBB224_8
; GFX90A-NEXT: .LBB224_4:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: .LBB224_5: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB224_5: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB224_7
+; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s6, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s6
@@ -17560,16 +17867,21 @@ define void @flat_atomic_fadd_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_add_f32_e32 v3, v1, v0
; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
-; GFX90A-NEXT: .LBB224_6: ; %Flow1
-; GFX90A-NEXT: s_cbranch_execnz .LBB224_8
-; GFX90A-NEXT: .LBB224_7: ; %atomicrmw.shared
+; GFX90A-NEXT: .LBB224_7: ; %Flow1
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: .LBB224_8: ; %Flow2
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB224_10
+; GFX90A-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: .LBB224_8: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB224_10: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v1
@@ -18294,31 +18606,38 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB235_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB235_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB235_4
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB235_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB235_5
-; GFX90A-NEXT: s_branch .LBB235_6
+; GFX90A-NEXT: s_branch .LBB235_5
; GFX90A-NEXT: .LBB235_3:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: s_branch .LBB235_7
+; GFX90A-NEXT: s_branch .LBB235_8
; GFX90A-NEXT: .LBB235_4:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB235_5: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB235_5: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB235_7
+; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s6, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s6
@@ -18330,9 +18649,14 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
; GFX90A-NEXT: buffer_store_dword v4, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v5, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB235_6: ; %Flow1
-; GFX90A-NEXT: s_cbranch_execnz .LBB235_8
-; GFX90A-NEXT: .LBB235_7: ; %atomicrmw.shared
+; GFX90A-NEXT: .LBB235_7: ; %Flow1
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: .LBB235_8: ; %Flow2
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB235_10
+; GFX90A-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
@@ -18340,7 +18664,7 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX90A-NEXT: .LBB235_8: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB235_10: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -18357,31 +18681,38 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB235_3
+; GFX950-NEXT: s_cbranch_scc0 .LBB235_3
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_vccz .LBB235_4
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc0 .LBB235_4
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc0
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB235_5
-; GFX950-NEXT: s_branch .LBB235_6
+; GFX950-NEXT: s_branch .LBB235_5
; GFX950-NEXT: .LBB235_3:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: s_branch .LBB235_7
+; GFX950-NEXT: s_branch .LBB235_8
; GFX950-NEXT: .LBB235_4:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB235_5: ; %atomicrmw.private
+; GFX950-NEXT: .LBB235_5: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB235_7
+; GFX950-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
@@ -18390,9 +18721,14 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[4:5], s2
-; GFX950-NEXT: .LBB235_6: ; %Flow1
-; GFX950-NEXT: s_cbranch_execnz .LBB235_8
-; GFX950-NEXT: .LBB235_7: ; %atomicrmw.shared
+; GFX950-NEXT: .LBB235_7: ; %Flow1
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: .LBB235_8: ; %Flow2
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB235_10
+; GFX950-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: v_mov_b32_e32 v2, s0
@@ -18400,7 +18736,7 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v0
; GFX950-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX950-NEXT: .LBB235_8: ; %atomicrmw.end
+; GFX950-NEXT: .LBB235_10: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -18421,29 +18757,36 @@ define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB236_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB236_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB236_4
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB236_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc
-; GFX90A-NEXT: s_cbranch_execz .LBB236_5
-; GFX90A-NEXT: s_branch .LBB236_6
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB236_5
; GFX90A-NEXT: .LBB236_3:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: s_branch .LBB236_7
+; GFX90A-NEXT: s_branch .LBB236_8
; GFX90A-NEXT: .LBB236_4:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX90A-NEXT: .LBB236_5: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB236_5: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB236_7
+; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s6, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s6
@@ -18453,16 +18796,21 @@ define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
; GFX90A-NEXT: buffer_store_dword v4, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v5, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB236_6: ; %Flow1
-; GFX90A-NEXT: s_cbranch_execnz .LBB236_8
-; GFX90A-NEXT: .LBB236_7: ; %atomicrmw.shared
+; GFX90A-NEXT: .LBB236_7: ; %Flow1
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: .LBB236_8: ; %Flow2
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB236_10
+; GFX90A-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s4
; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: .LBB236_8: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB236_10: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[2:3]
@@ -18476,45 +18824,57 @@ define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_shared_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB236_3
+; GFX950-NEXT: s_cbranch_scc0 .LBB236_3
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_vccz .LBB236_4
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc0 .LBB236_4
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc0
-; GFX950-NEXT: s_cbranch_execz .LBB236_5
-; GFX950-NEXT: s_branch .LBB236_6
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB236_5
; GFX950-NEXT: .LBB236_3:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: s_branch .LBB236_7
+; GFX950-NEXT: s_branch .LBB236_8
; GFX950-NEXT: .LBB236_4:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX950-NEXT: .LBB236_5: ; %atomicrmw.private
+; GFX950-NEXT: .LBB236_5: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB236_7
+; GFX950-NEXT: ; %bb.6: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
; GFX950-NEXT: scratch_store_dwordx2 off, v[4:5], s2
-; GFX950-NEXT: .LBB236_6: ; %Flow1
-; GFX950-NEXT: s_cbranch_execnz .LBB236_8
-; GFX950-NEXT: .LBB236_7: ; %atomicrmw.shared
+; GFX950-NEXT: .LBB236_7: ; %Flow1
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: .LBB236_8: ; %Flow2
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB236_10
+; GFX950-NEXT: ; %bb.9: ; %atomicrmw.shared
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, s0
; GFX950-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: .LBB236_8: ; %atomicrmw.end
+; GFX950-NEXT: .LBB236_10: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[2:3]
@@ -18538,11 +18898,11 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB237_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB237_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18595,11 +18955,11 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB237_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB237_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18654,12 +19014,12 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB238_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB238_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18705,12 +19065,12 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB238_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB238_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18764,23 +19124,29 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB239_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB239_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB239_3
-; GFX90A-NEXT: s_branch .LBB239_4
+; GFX90A-NEXT: s_branch .LBB239_3
; GFX90A-NEXT: .LBB239_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB239_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB239_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB239_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s4
@@ -18794,7 +19160,7 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB239_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB239_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -18811,23 +19177,29 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB239_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB239_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB239_3
-; GFX950-NEXT: s_branch .LBB239_4
+; GFX950-NEXT: s_branch .LBB239_3
; GFX950-NEXT: .LBB239_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB239_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB239_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB239_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -18838,7 +19210,7 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB239_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB239_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -18859,21 +19231,27 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB240_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB240_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB240_3
-; GFX90A-NEXT: s_branch .LBB240_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB240_3
; GFX90A-NEXT: .LBB240_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX90A-NEXT: .LBB240_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB240_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB240_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s4
@@ -18885,7 +19263,7 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB240_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB240_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[0:1]
@@ -18899,21 +19277,27 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB240_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB240_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB240_3
-; GFX950-NEXT: s_branch .LBB240_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB240_3
; GFX950-NEXT: .LBB240_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB240_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB240_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB240_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -18922,7 +19306,7 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB240_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB240_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -18946,23 +19330,29 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB241_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB241_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX90A-NEXT: s_cbranch_execz .LBB241_3
-; GFX90A-NEXT: s_branch .LBB241_4
+; GFX90A-NEXT: s_branch .LBB241_3
; GFX90A-NEXT: .LBB241_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX90A-NEXT: .LBB241_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB241_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB241_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s4
@@ -18976,7 +19366,7 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3
; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB241_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB241_5: ; %atomicrmw.end
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a[0:1]
; GFX90A-NEXT: ;;#ASMEND
@@ -18993,23 +19383,29 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB241_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB241_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_accvgpr_write_b32 a0, v2
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
-; GFX950-NEXT: s_cbranch_execz .LBB241_3
-; GFX950-NEXT: s_branch .LBB241_4
+; GFX950-NEXT: s_branch .LBB241_3
; GFX950-NEXT: .LBB241_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $agpr0_agpr1
-; GFX950-NEXT: .LBB241_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB241_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB241_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
@@ -19020,7 +19416,7 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]
; GFX950-NEXT: v_accvgpr_write_b32 a1, v3
; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
-; GFX950-NEXT: .LBB241_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB241_5: ; %atomicrmw.end
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use a[0:1]
; GFX950-NEXT: ;;#ASMEND
@@ -19041,21 +19437,27 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB242_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB242_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_execz .LBB242_3
-; GFX90A-NEXT: s_branch .LBB242_4
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_branch .LBB242_3
; GFX90A-NEXT: .LBB242_2:
+; GFX90A-NEXT: s_mov_b64 s[6:7], -1
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX90A-NEXT: .LBB242_3: ; %atomicrmw.private
+; GFX90A-NEXT: .LBB242_3: ; %Flow
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB242_5
+; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX90A-NEXT: s_cselect_b32 s4, s4, -1
; GFX90A-NEXT: v_mov_b32_e32 v6, s4
@@ -19067,7 +19469,7 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: .LBB242_4: ; %atomicrmw.end
+; GFX90A-NEXT: .LBB242_5: ; %atomicrmw.end
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use v[0:1]
@@ -19081,21 +19483,27 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB242_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB242_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cbranch_execz .LBB242_3
-; GFX950-NEXT: s_branch .LBB242_4
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: s_branch .LBB242_3
; GFX950-NEXT: .LBB242_2:
+; GFX950-NEXT: s_mov_b64 s[2:3], -1
; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-NEXT: .LBB242_3: ; %atomicrmw.private
+; GFX950-NEXT: .LBB242_3: ; %Flow
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc1 .LBB242_5
+; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
@@ -19104,7 +19512,7 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
-; GFX950-NEXT: .LBB242_4: ; %atomicrmw.end
+; GFX950-NEXT: .LBB242_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; use v[0:1]
@@ -19128,11 +19536,11 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB243_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB243_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19193,11 +19601,11 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB243_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB243_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19261,12 +19669,12 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB244_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB244_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19320,12 +19728,12 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB244_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB244_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19389,11 +19797,11 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB245_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB245_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19454,11 +19862,11 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB245_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB245_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19522,12 +19930,12 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB246_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB246_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19581,12 +19989,12 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB246_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB246_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/add.ll b/llvm/test/CodeGen/AMDGPU/add.ll
index b8814b64735e6..6e832df650204 100644
--- a/llvm/test/CodeGen/AMDGPU/add.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.ll
@@ -1157,15 +1157,22 @@ define amdgpu_kernel void @add64_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_ne_u64_e64 s[10:11], s[4:5], 0
; GFX6-NEXT: s_and_b64 vcc, exec, s[10:11]
-; GFX6-NEXT: s_cbranch_vccz .LBB9_4
+; GFX6-NEXT: s_cbranch_vccz .LBB9_2
; GFX6-NEXT: ; %bb.1: ; %else
; GFX6-NEXT: s_add_u32 s4, s4, s6
; GFX6-NEXT: s_addc_u32 s5, s5, s7
-; GFX6-NEXT: s_andn2_b64 vcc, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX6-NEXT: .LBB9_2: ; %if
+; GFX6-NEXT: s_branch .LBB9_3
+; GFX6-NEXT: .LBB9_2:
+; GFX6-NEXT: s_mov_b64 s[8:9], -1
+; GFX6-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX6-NEXT: .LBB9_3: ; %Flow
+; GFX6-NEXT: s_and_b64 s[6:7], s[8:9], exec
+; GFX6-NEXT: s_cselect_b32 s6, 1, 0
+; GFX6-NEXT: s_cmp_lg_u32 s6, 1
+; GFX6-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX6-NEXT: ; %bb.4: ; %if
; GFX6-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX6-NEXT: .LBB9_3: ; %endif
+; GFX6-NEXT: .LBB9_5: ; %endif
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -1173,9 +1180,6 @@ define amdgpu_kernel void @add64_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX6-NEXT: v_mov_b32_e32 v1, s5
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
-; GFX6-NEXT: .LBB9_4:
-; GFX6-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX6-NEXT: s_branch .LBB9_2
;
; GFX8-LABEL: add64_in_branch:
; GFX8: ; %bb.0: ; %entry
@@ -1183,15 +1187,22 @@ define amdgpu_kernel void @add64_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX8-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX8-NEXT: s_cbranch_scc0 .LBB9_2
; GFX8-NEXT: ; %bb.1: ; %else
; GFX8-NEXT: s_add_u32 s4, s4, s6
; GFX8-NEXT: s_addc_u32 s5, s5, s7
-; GFX8-NEXT: s_andn2_b64 vcc, exec, s[8:9]
-; GFX8-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX8-NEXT: .LBB9_2: ; %if
+; GFX8-NEXT: s_branch .LBB9_3
+; GFX8-NEXT: .LBB9_2:
+; GFX8-NEXT: s_mov_b64 s[8:9], -1
+; GFX8-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX8-NEXT: .LBB9_3: ; %Flow
+; GFX8-NEXT: s_and_b64 s[6:7], s[8:9], exec
+; GFX8-NEXT: s_cselect_b32 s6, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s6, 1
+; GFX8-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX8-NEXT: ; %bb.4: ; %if
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: .LBB9_3: ; %endif
+; GFX8-NEXT: .LBB9_5: ; %endif
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
@@ -1199,9 +1210,6 @@ define amdgpu_kernel void @add64_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
-; GFX8-NEXT: .LBB9_4:
-; GFX8-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX8-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: add64_in_branch:
; GFX9: ; %bb.0: ; %entry
@@ -1209,90 +1217,114 @@ define amdgpu_kernel void @add64_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u64 s[12:13], 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %else
; GFX9-NEXT: s_add_u32 s0, s12, s14
; GFX9-NEXT: s_addc_u32 s1, s13, s15
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX9-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %if
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[2:3], -1
+; GFX9-NEXT: ; implicit-def: $sgpr0_sgpr1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s2, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %if
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[10:11], 0x0
-; GFX9-NEXT: .LBB9_3: ; %endif
+; GFX9-NEXT: .LBB9_5: ; %endif
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX9-NEXT: s_endpgm
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: ; implicit-def: $sgpr0_sgpr1
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX10-LABEL: add64_in_branch:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u64 s[12:13], 0
-; GFX10-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX10-NEXT: s_cbranch_scc0 .LBB9_2
; GFX10-NEXT: ; %bb.1: ; %else
; GFX10-NEXT: s_add_u32 s0, s12, s14
; GFX10-NEXT: s_addc_u32 s1, s13, s15
-; GFX10-NEXT: s_cbranch_execnz .LBB9_3
-; GFX10-NEXT: .LBB9_2: ; %if
+; GFX10-NEXT: s_mov_b32 s2, 0
+; GFX10-NEXT: s_branch .LBB9_3
+; GFX10-NEXT: .LBB9_2:
+; GFX10-NEXT: s_mov_b32 s2, -1
+; GFX10-NEXT: ; implicit-def: $sgpr0_sgpr1
+; GFX10-NEXT: .LBB9_3: ; %Flow
+; GFX10-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10-NEXT: s_cselect_b32 s2, 1, 0
+; GFX10-NEXT: s_cmp_lg_u32 s2, 1
+; GFX10-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX10-NEXT: ; %bb.4: ; %if
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[10:11], 0x0
-; GFX10-NEXT: .LBB9_3: ; %endif
+; GFX10-NEXT: .LBB9_5: ; %endif
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX10-NEXT: s_endpgm
-; GFX10-NEXT: .LBB9_4:
-; GFX10-NEXT: ; implicit-def: $sgpr0_sgpr1
-; GFX10-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: add64_in_branch:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB9_2
; GFX11-NEXT: ; %bb.1: ; %else
; GFX11-NEXT: s_add_u32 s4, s4, s6
; GFX11-NEXT: s_addc_u32 s5, s5, s7
-; GFX11-NEXT: s_cbranch_execnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %if
+; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_branch .LBB9_3
+; GFX11-NEXT: .LBB9_2:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX11-NEXT: .LBB9_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX11-NEXT: ; %bb.4: ; %if
; GFX11-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX11-NEXT: .LBB9_3: ; %endif
+; GFX11-NEXT: .LBB9_5: ; %endif
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, s4
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX11-NEXT: s_branch .LBB9_2
;
; GFX12-LABEL: add64_in_branch:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX12-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX12-NEXT: s_cbranch_scc0 .LBB9_2
; GFX12-NEXT: ; %bb.1: ; %else
; GFX12-NEXT: s_add_nc_u64 s[4:5], s[4:5], s[6:7]
-; GFX12-NEXT: s_cbranch_execnz .LBB9_3
-; GFX12-NEXT: .LBB9_2: ; %if
+; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_branch .LBB9_3
+; GFX12-NEXT: .LBB9_2:
+; GFX12-NEXT: s_mov_b32 s6, -1
+; GFX12-NEXT: ; implicit-def: $sgpr4_sgpr5
+; GFX12-NEXT: .LBB9_3: ; %Flow
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX12-NEXT: ; %bb.4: ; %if
; GFX12-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX12-NEXT: .LBB9_3: ; %endif
+; GFX12-NEXT: .LBB9_5: ; %endif
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
-; GFX12-NEXT: .LBB9_4:
-; GFX12-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX12-NEXT: s_branch .LBB9_2
entry:
%0 = icmp eq i64 %a, 0
br i1 %0, label %if, label %else
diff --git a/llvm/test/CodeGen/AMDGPU/add_i1.ll b/llvm/test/CodeGen/AMDGPU/add_i1.ll
index ac5386bdfc06f..a7a63f2d8d551 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i1.ll
@@ -69,7 +69,9 @@ define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1
; GFX9-NEXT: v_and_b32_e32 v1, 1, v1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GFX9-NEXT: s_xor_b64 s[2:3], vcc, -1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_store_byte v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -83,7 +85,9 @@ define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1
; GFX10-NEXT: v_and_b32_e32 v1, 1, v1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX10-NEXT: s_xor_b32 s2, vcc_lo, -1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10-NEXT: s_cselect_b32 s2, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: global_store_byte v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -98,7 +102,10 @@ define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX11-NEXT: s_xor_b32 s2, vcc_lo, -1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
; GFX11-NEXT: global_store_b8 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
%a = load volatile i1, ptr addrspace(1) %in
diff --git a/llvm/test/CodeGen/AMDGPU/addsub64_carry.ll b/llvm/test/CodeGen/AMDGPU/addsub64_carry.ll
index c26f1a4d93802..30e275a02b334 100644
--- a/llvm/test/CodeGen/AMDGPU/addsub64_carry.ll
+++ b/llvm/test/CodeGen/AMDGPU/addsub64_carry.ll
@@ -197,19 +197,17 @@ define amdgpu_ps <2 x i64> @s_uadd_v2i64(<2 x i64> inreg %val0, <2 x i64> inreg
; CHECK-NEXT: s_add_u32 s6, s2, s6
; CHECK-NEXT: s_addc_u32 s7, s3, s7
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: s_add_u32 s0, s0, s4
-; CHECK-NEXT: s_addc_u32 s1, s1, s5
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_add_u32 s4, s0, s4
+; CHECK-NEXT: s_addc_u32 s5, s1, s5
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v7
-; CHECK-NEXT: v_readfirstlane_b32 s2, v6
+; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: v_mov_b32_e32 v2, s4
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mov_b32_e32 v4, s6
; CHECK-NEXT: v_mov_b32_e32 v5, s7
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s3, s2
+; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
@@ -227,19 +225,17 @@ define amdgpu_ps <2 x i64> @s_usub_v2i64(<2 x i64> inreg %val0, <2 x i64> inreg
; CHECK-NEXT: s_sub_u32 s6, s2, s6
; CHECK-NEXT: s_subb_u32 s7, s3, s7
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: s_sub_u32 s0, s0, s4
-; CHECK-NEXT: s_subb_u32 s1, s1, s5
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_sub_u32 s4, s0, s4
+; CHECK-NEXT: s_subb_u32 s5, s1, s5
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v7
-; CHECK-NEXT: v_readfirstlane_b32 s2, v6
+; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: v_mov_b32_e32 v2, s4
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mov_b32_e32 v4, s6
; CHECK-NEXT: v_mov_b32_e32 v5, s7
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s3, s2
+; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
@@ -254,15 +250,15 @@ define amdgpu_ps <2 x i64> @s_usub_v2i64(<2 x i64> inreg %val0, <2 x i64> inreg
define amdgpu_ps i64 @s_uadd_i64(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
; CHECK-LABEL: s_uadd_i64:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_add_u32 s0, s0, s2
-; CHECK-NEXT: s_addc_u32 s1, s1, s3
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_add_u32 s2, s0, s2
+; CHECK-NEXT: s_addc_u32 s3, s1, s3
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
%pair = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %val0, i64 %val1)
@@ -276,15 +272,15 @@ define amdgpu_ps i64 @s_uadd_i64(i64 inreg %val0, i64 inreg %val1, ptr %ptrval)
define amdgpu_ps i64 @s_uadd_p1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
; CHECK-LABEL: s_uadd_p1:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_add_u32 s0, s0, 1
-; CHECK-NEXT: s_addc_u32 s1, s1, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_add_u32 s2, s0, 1
+; CHECK-NEXT: s_addc_u32 s3, s1, 0
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
%pair = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %val0, i64 1)
@@ -298,15 +294,15 @@ define amdgpu_ps i64 @s_uadd_p1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
define amdgpu_ps i64 @s_uadd_n1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
; CHECK-LABEL: s_uadd_n1:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_add_u32 s0, s0, -1
-; CHECK-NEXT: s_addc_u32 s1, s1, -1
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_add_u32 s2, s0, -1
+; CHECK-NEXT: s_addc_u32 s3, s1, -1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
%pair = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %val0, i64 -1)
@@ -320,15 +316,15 @@ define amdgpu_ps i64 @s_uadd_n1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
define amdgpu_ps i64 @s_usub_p1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
; CHECK-LABEL: s_usub_p1:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_sub_u32 s0, s0, 1
-; CHECK-NEXT: s_subb_u32 s1, s1, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_sub_u32 s2, s0, 1
+; CHECK-NEXT: s_subb_u32 s3, s1, 0
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
%pair = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %val0, i64 1)
@@ -342,15 +338,15 @@ define amdgpu_ps i64 @s_usub_p1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
define amdgpu_ps i64 @s_usub_n1(i64 inreg %val0, i64 inreg %val1, ptr %ptrval) {
; CHECK-LABEL: s_usub_n1:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_sub_u32 s0, s0, -1
-; CHECK-NEXT: s_subb_u32 s1, s1, -1
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: v_mov_b32_e32 v3, s1
+; CHECK-NEXT: s_sub_u32 s2, s0, -1
+; CHECK-NEXT: s_subb_u32 s3, s1, -1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ; return to shader part epilog
%pair = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %val0, i64 -1)
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 727728145dfe0..bd4e24d96b612 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -512,10 +512,10 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
; GFX908-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX908-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
+; GFX908-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
; GFX908-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX908-NEXT: s_mov_b32 s12, 0
-; GFX908-NEXT: s_mov_b32 s9, s12
+; GFX908-NEXT: s_mov_b32 s10, 0
+; GFX908-NEXT: s_mov_b32 s9, s10
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cvt_f32_u32_e32 v0, s7
; GFX908-NEXT: s_sub_i32 s1, 0, s7
@@ -525,93 +525,97 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: v_mov_b32_e32 v1, 0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX908-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX908-NEXT: v_readfirstlane_b32 s2, v0
-; GFX908-NEXT: s_mul_i32 s1, s1, s2
-; GFX908-NEXT: s_mul_hi_u32 s1, s2, s1
-; GFX908-NEXT: s_add_i32 s2, s2, s1
-; GFX908-NEXT: s_mul_hi_u32 s1, s6, s2
-; GFX908-NEXT: s_mul_i32 s2, s1, s7
-; GFX908-NEXT: s_sub_i32 s2, s6, s2
-; GFX908-NEXT: s_add_i32 s3, s1, 1
-; GFX908-NEXT: s_sub_i32 s6, s2, s7
-; GFX908-NEXT: s_cmp_ge_u32 s2, s7
-; GFX908-NEXT: s_cselect_b32 s1, s3, s1
-; GFX908-NEXT: s_cselect_b32 s2, s6, s2
-; GFX908-NEXT: s_add_i32 s3, s1, 1
-; GFX908-NEXT: s_cmp_ge_u32 s2, s7
-; GFX908-NEXT: s_cselect_b32 s8, s3, s1
-; GFX908-NEXT: s_lshr_b32 s2, s0, 16
-; GFX908-NEXT: v_cvt_f32_f16_e32 v19, s2
+; GFX908-NEXT: v_readfirstlane_b32 s8, v0
+; GFX908-NEXT: s_mul_i32 s1, s1, s8
+; GFX908-NEXT: s_mul_hi_u32 s1, s8, s1
+; GFX908-NEXT: s_add_i32 s8, s8, s1
+; GFX908-NEXT: s_mul_hi_u32 s1, s6, s8
+; GFX908-NEXT: s_mul_i32 s8, s1, s7
+; GFX908-NEXT: s_sub_i32 s6, s6, s8
+; GFX908-NEXT: s_add_i32 s11, s1, 1
+; GFX908-NEXT: s_sub_i32 s8, s6, s7
+; GFX908-NEXT: s_cmp_ge_u32 s6, s7
+; GFX908-NEXT: s_cselect_b32 s1, s11, s1
+; GFX908-NEXT: s_cselect_b32 s6, s8, s6
+; GFX908-NEXT: s_add_i32 s8, s1, 1
+; GFX908-NEXT: s_cmp_ge_u32 s6, s7
+; GFX908-NEXT: s_cselect_b32 s8, s8, s1
+; GFX908-NEXT: s_lshr_b32 s11, s0, 16
+; GFX908-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
+; GFX908-NEXT: v_cvt_f32_f16_e32 v19, s11
; GFX908-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX908-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
+; GFX908-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX908-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s2, v16
-; GFX908-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX908-NEXT: s_mul_i32 s3, s5, s2
-; GFX908-NEXT: s_mul_hi_u32 s5, s4, s2
-; GFX908-NEXT: s_mul_i32 s2, s4, s2
-; GFX908-NEXT: s_add_i32 s3, s5, s3
-; GFX908-NEXT: s_lshl_b64 s[4:5], s[2:3], 5
+; GFX908-NEXT: v_readfirstlane_b32 s9, v16
+; GFX908-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX908-NEXT: s_mul_i32 s5, s5, s9
+; GFX908-NEXT: s_mul_hi_u32 s11, s4, s9
+; GFX908-NEXT: s_mul_i32 s4, s4, s9
+; GFX908-NEXT: s_add_i32 s5, s11, s5
+; GFX908-NEXT: s_lshl_b64 s[4:5], s[4:5], 5
; GFX908-NEXT: s_branch .LBB3_2
; GFX908-NEXT: .LBB3_1: ; %Flow20
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX908-NEXT: s_cbranch_vccz .LBB3_12
+; GFX908-NEXT: s_and_b64 s[16:17], s[16:17], exec
+; GFX908-NEXT: s_cselect_b32 s9, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s9, 1
+; GFX908-NEXT: s_cbranch_scc0 .LBB3_14
; GFX908-NEXT: .LBB3_2: ; %bb9
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB3_5 Depth 2
+; GFX908-NEXT: ; Child Loop BB3_6 Depth 2
; GFX908-NEXT: s_mov_b64 s[18:19], -1
; GFX908-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX908-NEXT: s_cbranch_vccz .LBB3_10
+; GFX908-NEXT: s_cbranch_vccz .LBB3_12
; GFX908-NEXT: ; %bb.3: ; %bb14
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
-; GFX908-NEXT: s_cmp_lt_i32 s11, 0
-; GFX908-NEXT: s_mov_b32 s13, s12
-; GFX908-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX908-NEXT: s_cmp_gt_i32 s11, -1
-; GFX908-NEXT: v_mov_b32_e32 v4, s12
-; GFX908-NEXT: v_mov_b32_e32 v6, s12
-; GFX908-NEXT: v_mov_b32_e32 v8, s12
-; GFX908-NEXT: v_mov_b32_e32 v5, s13
-; GFX908-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX908-NEXT: v_mov_b32_e32 v7, s13
-; GFX908-NEXT: v_mov_b32_e32 v9, s13
-; GFX908-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[2:3]
-; GFX908-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v10
+; GFX908-NEXT: s_cmp_lt_i32 s3, 0
+; GFX908-NEXT: s_mov_b32 s11, s10
+; GFX908-NEXT: s_cselect_b64 s[16:17], -1, 0
+; GFX908-NEXT: s_cmp_gt_i32 s3, -1
+; GFX908-NEXT: v_mov_b32_e32 v4, s10
+; GFX908-NEXT: v_mov_b32_e32 v6, s10
+; GFX908-NEXT: v_mov_b32_e32 v8, s10
+; GFX908-NEXT: v_mov_b32_e32 v5, s11
+; GFX908-NEXT: s_cselect_b64 s[20:21], -1, 0
+; GFX908-NEXT: v_mov_b32_e32 v7, s11
+; GFX908-NEXT: v_mov_b32_e32 v9, s11
; GFX908-NEXT: v_mov_b32_e32 v11, v5
-; GFX908-NEXT: s_mov_b64 s[20:21], s[14:15]
+; GFX908-NEXT: s_mov_b64 s[18:19], s[12:13]
; GFX908-NEXT: v_mov_b32_e32 v10, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s9, v2
-; GFX908-NEXT: v_readfirstlane_b32 s13, v3
+; GFX908-NEXT: v_readfirstlane_b32 s11, v3
; GFX908-NEXT: s_add_u32 s9, s9, 1
-; GFX908-NEXT: s_addc_u32 s13, s13, 0
+; GFX908-NEXT: s_addc_u32 s11, s11, 0
; GFX908-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX908-NEXT: s_mul_i32 s13, s6, s13
+; GFX908-NEXT: s_mul_i32 s11, s6, s11
; GFX908-NEXT: s_mul_i32 s23, s7, s9
-; GFX908-NEXT: s_add_i32 s13, s22, s13
+; GFX908-NEXT: s_add_i32 s11, s22, s11
; GFX908-NEXT: s_mul_i32 s9, s6, s9
-; GFX908-NEXT: s_add_i32 s13, s13, s23
-; GFX908-NEXT: s_branch .LBB3_5
+; GFX908-NEXT: s_add_i32 s11, s11, s23
+; GFX908-NEXT: s_branch .LBB3_6
; GFX908-NEXT: .LBB3_4: ; %bb58
-; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
; GFX908-NEXT: v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX908-NEXT: s_add_u32 s20, s20, s4
-; GFX908-NEXT: s_addc_u32 s21, s21, s5
+; GFX908-NEXT: s_add_u32 s18, s18, s4
+; GFX908-NEXT: s_addc_u32 s19, s19, s5
; GFX908-NEXT: s_mov_b64 s[22:23], 0
; GFX908-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v3
-; GFX908-NEXT: s_andn2_b64 vcc, exec, s[24:25]
-; GFX908-NEXT: s_cbranch_vccz .LBB3_9
-; GFX908-NEXT: .LBB3_5: ; %bb16
+; GFX908-NEXT: .LBB3_5: ; %Flow18
+; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
+; GFX908-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; GFX908-NEXT: s_cselect_b32 s24, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s24, 1
+; GFX908-NEXT: s_cbranch_scc0 .LBB3_11
+; GFX908-NEXT: .LBB3_6: ; %bb16
; GFX908-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: s_add_u32 s22, s20, s9
-; GFX908-NEXT: s_addc_u32 s23, s21, s13
+; GFX908-NEXT: s_add_u32 s22, s18, s9
+; GFX908-NEXT: s_addc_u32 s23, s19, s11
; GFX908-NEXT: global_load_dword v21, v17, s[22:23] offset:16 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: global_load_dword v20, v17, s[22:23] offset:20 glc
@@ -622,11 +626,13 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: ds_read_b64 v[12:13], v17
; GFX908-NEXT: ds_read_b64 v[14:15], v0
-; GFX908-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GFX908-NEXT: s_and_b64 s[22:23], s[20:21], exec
+; GFX908-NEXT: s_cselect_b32 s22, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s22, 1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_cbranch_vccnz .LBB3_7
-; GFX908-NEXT: ; %bb.6: ; %bb51
-; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: s_cbranch_scc1 .LBB3_8
+; GFX908-NEXT: ; %bb.7: ; %bb51
+; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
; GFX908-NEXT: v_add_f32_e32 v22, v18, v12
; GFX908-NEXT: v_add_f32_e32 v23, v19, v13
; GFX908-NEXT: v_add_f32_e32 v24, 0, v12
@@ -643,42 +649,49 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: v_add_f32_e32 v8, v8, v14
; GFX908-NEXT: v_add_f32_e32 v11, v11, v13
; GFX908-NEXT: v_add_f32_e32 v10, v10, v12
-; GFX908-NEXT: s_branch .LBB3_4
-; GFX908-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX908-NEXT: s_mov_b64 s[22:23], s[18:19]
-; GFX908-NEXT: s_andn2_b64 vcc, exec, s[22:23]
-; GFX908-NEXT: s_cbranch_vccz .LBB3_4
-; GFX908-NEXT: ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
+; GFX908-NEXT: s_mov_b64 s[22:23], -1
+; GFX908-NEXT: s_branch .LBB3_9
+; GFX908-NEXT: .LBB3_8: ; in Loop: Header=BB3_6 Depth=2
+; GFX908-NEXT: s_mov_b64 s[22:23], s[16:17]
+; GFX908-NEXT: .LBB3_9: ; %Flow
+; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
+; GFX908-NEXT: s_and_b64 s[22:23], s[22:23], exec
+; GFX908-NEXT: s_cselect_b32 s22, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s22, 1
+; GFX908-NEXT: s_cbranch_scc0 .LBB3_4
+; GFX908-NEXT: ; %bb.10: ; in Loop: Header=BB3_6 Depth=2
; GFX908-NEXT: s_mov_b64 s[22:23], -1
; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX908-NEXT: ; implicit-def: $sgpr20_sgpr21
-; GFX908-NEXT: .LBB3_9: ; %loop.exit.guard
+; GFX908-NEXT: ; implicit-def: $sgpr18_sgpr19
+; GFX908-NEXT: s_mov_b64 s[24:25], -1
+; GFX908-NEXT: s_branch .LBB3_5
+; GFX908-NEXT: .LBB3_11: ; %loop.exit.guard
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX908-NEXT: .LBB3_10: ; %Flow19
+; GFX908-NEXT: .LBB3_12: ; %Flow19
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_mov_b64 s[2:3], -1
+; GFX908-NEXT: s_mov_b64 s[16:17], -1
; GFX908-NEXT: s_and_b64 vcc, exec, s[18:19]
; GFX908-NEXT: s_cbranch_vccz .LBB3_1
-; GFX908-NEXT: ; %bb.11: ; %bb12
+; GFX908-NEXT: ; %bb.13: ; %bb12
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_add_u32 s10, s10, s8
-; GFX908-NEXT: s_addc_u32 s11, s11, 0
-; GFX908-NEXT: s_add_u32 s14, s14, s16
-; GFX908-NEXT: s_addc_u32 s15, s15, s17
-; GFX908-NEXT: s_mov_b64 s[2:3], 0
+; GFX908-NEXT: s_add_u32 s2, s2, s8
+; GFX908-NEXT: s_addc_u32 s3, s3, 0
+; GFX908-NEXT: s_add_u32 s12, s12, s14
+; GFX908-NEXT: s_addc_u32 s13, s13, s15
+; GFX908-NEXT: s_mov_b64 s[16:17], 0
; GFX908-NEXT: s_branch .LBB3_1
-; GFX908-NEXT: .LBB3_12: ; %DummyReturnBlock
+; GFX908-NEXT: .LBB3_14: ; %DummyReturnBlock
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: introduced_copy_to_sgpr:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX90A-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
; GFX90A-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX90A-NEXT: s_mov_b32 s12, 0
-; GFX90A-NEXT: s_mov_b32 s9, s12
+; GFX90A-NEXT: s_mov_b32 s10, 0
+; GFX90A-NEXT: s_mov_b32 s9, s10
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s7
; GFX90A-NEXT: s_sub_i32 s1, 0, s7
@@ -688,88 +701,92 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX90A-NEXT: v_cvt_u32_f32_e32 v1, v0
; GFX90A-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX90A-NEXT: v_readfirstlane_b32 s2, v1
-; GFX90A-NEXT: s_mul_i32 s1, s1, s2
-; GFX90A-NEXT: s_mul_hi_u32 s1, s2, s1
-; GFX90A-NEXT: s_add_i32 s2, s2, s1
-; GFX90A-NEXT: s_mul_hi_u32 s1, s6, s2
-; GFX90A-NEXT: s_mul_i32 s2, s1, s7
-; GFX90A-NEXT: s_sub_i32 s2, s6, s2
-; GFX90A-NEXT: s_add_i32 s3, s1, 1
-; GFX90A-NEXT: s_sub_i32 s6, s2, s7
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s7
-; GFX90A-NEXT: s_cselect_b32 s1, s3, s1
-; GFX90A-NEXT: s_cselect_b32 s2, s6, s2
-; GFX90A-NEXT: s_add_i32 s3, s1, 1
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s7
-; GFX90A-NEXT: s_cselect_b32 s8, s3, s1
-; GFX90A-NEXT: s_lshr_b32 s2, s0, 16
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v1, s2
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: s_mul_i32 s1, s1, s8
+; GFX90A-NEXT: s_mul_hi_u32 s1, s8, s1
+; GFX90A-NEXT: s_add_i32 s8, s8, s1
+; GFX90A-NEXT: s_mul_hi_u32 s1, s6, s8
+; GFX90A-NEXT: s_mul_i32 s8, s1, s7
+; GFX90A-NEXT: s_sub_i32 s6, s6, s8
+; GFX90A-NEXT: s_add_i32 s11, s1, 1
+; GFX90A-NEXT: s_sub_i32 s8, s6, s7
+; GFX90A-NEXT: s_cmp_ge_u32 s6, s7
+; GFX90A-NEXT: s_cselect_b32 s1, s11, s1
+; GFX90A-NEXT: s_cselect_b32 s6, s8, s6
+; GFX90A-NEXT: s_add_i32 s8, s1, 1
+; GFX90A-NEXT: s_cmp_ge_u32 s6, s7
+; GFX90A-NEXT: s_cselect_b32 s8, s8, s1
+; GFX90A-NEXT: s_lshr_b32 s11, s0, 16
+; GFX90A-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v1, s11
; GFX90A-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX90A-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
+; GFX90A-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX90A-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s2, v18
-; GFX90A-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX90A-NEXT: s_mul_i32 s3, s5, s2
-; GFX90A-NEXT: s_mul_hi_u32 s5, s4, s2
-; GFX90A-NEXT: s_mul_i32 s2, s4, s2
-; GFX90A-NEXT: s_add_i32 s3, s5, s3
-; GFX90A-NEXT: s_lshl_b64 s[4:5], s[2:3], 5
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v18
+; GFX90A-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX90A-NEXT: s_mul_i32 s5, s5, s9
+; GFX90A-NEXT: s_mul_hi_u32 s11, s4, s9
+; GFX90A-NEXT: s_mul_i32 s4, s4, s9
+; GFX90A-NEXT: s_add_i32 s5, s11, s5
+; GFX90A-NEXT: s_lshl_b64 s[4:5], s[4:5], 5
; GFX90A-NEXT: s_branch .LBB3_2
; GFX90A-NEXT: .LBB3_1: ; %Flow20
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX90A-NEXT: s_cbranch_vccz .LBB3_12
+; GFX90A-NEXT: s_and_b64 s[16:17], s[16:17], exec
+; GFX90A-NEXT: s_cselect_b32 s9, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s9, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB3_14
; GFX90A-NEXT: .LBB3_2: ; %bb9
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB3_5 Depth 2
+; GFX90A-NEXT: ; Child Loop BB3_6 Depth 2
; GFX90A-NEXT: s_mov_b64 s[18:19], -1
; GFX90A-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX90A-NEXT: s_cbranch_vccz .LBB3_10
+; GFX90A-NEXT: s_cbranch_vccz .LBB3_12
; GFX90A-NEXT: ; %bb.3: ; %bb14
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX90A-NEXT: s_cmp_lt_i32 s11, 0
-; GFX90A-NEXT: s_mov_b32 s13, s12
-; GFX90A-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX90A-NEXT: s_cmp_gt_i32 s11, -1
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[2:3]
-; GFX90A-NEXT: s_mov_b64 s[20:21], s[14:15]
-; GFX90A-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v12
+; GFX90A-NEXT: s_cmp_lt_i32 s3, 0
+; GFX90A-NEXT: s_mov_b32 s11, s10
+; GFX90A-NEXT: s_cselect_b64 s[16:17], -1, 0
+; GFX90A-NEXT: s_cmp_gt_i32 s3, -1
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-NEXT: s_cselect_b64 s[20:21], -1, 0
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-NEXT: s_mov_b64 s[18:19], s[12:13]
; GFX90A-NEXT: v_pk_mov_b32 v[12:13], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s9, v4
-; GFX90A-NEXT: v_readfirstlane_b32 s13, v5
+; GFX90A-NEXT: v_readfirstlane_b32 s11, v5
; GFX90A-NEXT: s_add_u32 s9, s9, 1
-; GFX90A-NEXT: s_addc_u32 s13, s13, 0
+; GFX90A-NEXT: s_addc_u32 s11, s11, 0
; GFX90A-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX90A-NEXT: s_mul_i32 s13, s6, s13
+; GFX90A-NEXT: s_mul_i32 s11, s6, s11
; GFX90A-NEXT: s_mul_i32 s23, s7, s9
-; GFX90A-NEXT: s_add_i32 s13, s22, s13
+; GFX90A-NEXT: s_add_i32 s11, s22, s11
; GFX90A-NEXT: s_mul_i32 s9, s6, s9
-; GFX90A-NEXT: s_add_i32 s13, s13, s23
-; GFX90A-NEXT: s_branch .LBB3_5
+; GFX90A-NEXT: s_add_i32 s11, s11, s23
+; GFX90A-NEXT: s_branch .LBB3_6
; GFX90A-NEXT: .LBB3_4: ; %bb58
-; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
; GFX90A-NEXT: v_add_co_u32_sdwa v4, vcc, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX90A-NEXT: s_add_u32 s20, s20, s4
-; GFX90A-NEXT: s_addc_u32 s21, s21, s5
+; GFX90A-NEXT: s_add_u32 s18, s18, s4
+; GFX90A-NEXT: s_addc_u32 s19, s19, s5
; GFX90A-NEXT: s_mov_b64 s[22:23], 0
; GFX90A-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v5
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[24:25]
-; GFX90A-NEXT: s_cbranch_vccz .LBB3_9
-; GFX90A-NEXT: .LBB3_5: ; %bb16
+; GFX90A-NEXT: .LBB3_5: ; %Flow18
+; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
+; GFX90A-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; GFX90A-NEXT: s_cselect_b32 s24, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s24, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB3_11
+; GFX90A-NEXT: .LBB3_6: ; %bb16
; GFX90A-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT: s_add_u32 s22, s20, s9
-; GFX90A-NEXT: s_addc_u32 s23, s21, s13
+; GFX90A-NEXT: s_add_u32 s22, s18, s9
+; GFX90A-NEXT: s_addc_u32 s23, s19, s11
; GFX90A-NEXT: global_load_dword v21, v19, s[22:23] offset:16 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: global_load_dword v20, v19, s[22:23] offset:20 glc
@@ -780,12 +797,14 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: ds_read_b64 v[14:15], v19
; GFX90A-NEXT: ds_read_b64 v[16:17], v0
-; GFX90A-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX90A-NEXT: ; kill: killed $sgpr22 killed $sgpr23
+; GFX90A-NEXT: s_and_b64 s[22:23], s[20:21], exec
+; GFX90A-NEXT: s_cselect_b32 s22, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s22, 1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_vccnz .LBB3_7
-; GFX90A-NEXT: ; %bb.6: ; %bb51
-; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: s_cbranch_scc1 .LBB3_8
+; GFX90A-NEXT: ; %bb.7: ; %bb51
+; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
; GFX90A-NEXT: v_cvt_f32_f16_sdwa v23, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX90A-NEXT: v_cvt_f32_f16_e32 v22, v21
; GFX90A-NEXT: v_cvt_f32_f16_sdwa v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
@@ -798,32 +817,39 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[26:27]
; GFX90A-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[16:17]
; GFX90A-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[14:15]
-; GFX90A-NEXT: s_branch .LBB3_4
-; GFX90A-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX90A-NEXT: s_mov_b64 s[22:23], s[18:19]
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[22:23]
-; GFX90A-NEXT: s_cbranch_vccz .LBB3_4
-; GFX90A-NEXT: ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
+; GFX90A-NEXT: s_mov_b64 s[22:23], -1
+; GFX90A-NEXT: s_branch .LBB3_9
+; GFX90A-NEXT: .LBB3_8: ; in Loop: Header=BB3_6 Depth=2
+; GFX90A-NEXT: s_mov_b64 s[22:23], s[16:17]
+; GFX90A-NEXT: .LBB3_9: ; %Flow
+; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
+; GFX90A-NEXT: s_and_b64 s[22:23], s[22:23], exec
+; GFX90A-NEXT: s_cselect_b32 s22, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s22, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB3_4
+; GFX90A-NEXT: ; %bb.10: ; in Loop: Header=BB3_6 Depth=2
; GFX90A-NEXT: s_mov_b64 s[22:23], -1
; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX90A-NEXT: ; implicit-def: $sgpr20_sgpr21
-; GFX90A-NEXT: .LBB3_9: ; %loop.exit.guard
+; GFX90A-NEXT: ; implicit-def: $sgpr18_sgpr19
+; GFX90A-NEXT: s_mov_b64 s[24:25], -1
+; GFX90A-NEXT: s_branch .LBB3_5
+; GFX90A-NEXT: .LBB3_11: ; %loop.exit.guard
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX90A-NEXT: .LBB3_10: ; %Flow19
+; GFX90A-NEXT: .LBB3_12: ; %Flow19
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_mov_b64 s[2:3], -1
+; GFX90A-NEXT: s_mov_b64 s[16:17], -1
; GFX90A-NEXT: s_and_b64 vcc, exec, s[18:19]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_1
-; GFX90A-NEXT: ; %bb.11: ; %bb12
+; GFX90A-NEXT: ; %bb.13: ; %bb12
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_add_u32 s10, s10, s8
-; GFX90A-NEXT: s_addc_u32 s11, s11, 0
-; GFX90A-NEXT: s_add_u32 s14, s14, s16
-; GFX90A-NEXT: s_addc_u32 s15, s15, s17
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
+; GFX90A-NEXT: s_add_u32 s2, s2, s8
+; GFX90A-NEXT: s_addc_u32 s3, s3, 0
+; GFX90A-NEXT: s_add_u32 s12, s12, s14
+; GFX90A-NEXT: s_addc_u32 s13, s13, s15
+; GFX90A-NEXT: s_mov_b64 s[16:17], 0
; GFX90A-NEXT: s_branch .LBB3_1
-; GFX90A-NEXT: .LBB3_12: ; %DummyReturnBlock
+; GFX90A-NEXT: .LBB3_14: ; %DummyReturnBlock
; GFX90A-NEXT: s_endpgm
bb:
%i = load volatile i16, ptr addrspace(4) poison, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2d5fc5cf22425..0960d74b18522 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -251,10 +251,18 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -287,7 +295,7 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -321,8 +329,6 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v32i32_to_v32f32_scalar:
; VI: ; %bb.0:
@@ -347,10 +353,18 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -383,7 +397,7 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -417,8 +431,6 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v32i32_to_v32f32_scalar:
; GFX9: ; %bb.0:
@@ -443,10 +455,18 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -479,7 +499,7 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -513,8 +533,6 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v32i32_to_v32f32_scalar:
; GFX11: ; %bb.0:
@@ -536,11 +554,16 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -573,7 +596,7 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -592,8 +615,6 @@ define inreg <32 x float> @bitcast_v32i32_to_v32f32_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -872,10 +893,18 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v31, s67, 1.0
; SI-NEXT: v_add_f32_e64 v30, s66, 1.0
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -908,10 +937,8 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -944,7 +971,7 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -1023,10 +1050,18 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v31, s67, 1.0
; VI-NEXT: v_add_f32_e64 v30, s66, 1.0
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -1059,10 +1094,8 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -1095,7 +1128,7 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -1174,10 +1207,18 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX9-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -1210,10 +1251,8 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -1246,7 +1285,7 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -1326,11 +1365,16 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX11-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -1364,8 +1408,6 @@ define inreg <32 x i32> @bitcast_v32f32_to_v32i32_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1665,10 +1707,18 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1701,7 +1751,7 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1735,8 +1785,6 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v32i32_to_v16i64_scalar:
; VI: ; %bb.0:
@@ -1761,10 +1809,18 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1797,7 +1853,7 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1831,8 +1887,6 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v32i32_to_v16i64_scalar:
; GFX9: ; %bb.0:
@@ -1857,10 +1911,18 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1893,7 +1955,7 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1927,8 +1989,6 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v32i32_to_v16i64_scalar:
; GFX11: ; %bb.0:
@@ -1950,11 +2010,16 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1987,7 +2052,7 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2006,8 +2071,6 @@ define inreg <16 x i64> @bitcast_v32i32_to_v16i64_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2276,10 +2339,18 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -2312,7 +2383,7 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2346,8 +2417,6 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v16i64_to_v32i32_scalar:
; VI: ; %bb.0:
@@ -2372,10 +2441,18 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -2408,7 +2485,7 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2442,8 +2519,6 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v16i64_to_v32i32_scalar:
; GFX9: ; %bb.0:
@@ -2468,10 +2543,18 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -2504,7 +2587,7 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2538,8 +2621,6 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v16i64_to_v32i32_scalar:
; GFX11: ; %bb.0:
@@ -2561,11 +2642,16 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -2598,7 +2684,7 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2617,8 +2703,6 @@ define inreg <32 x i32> @bitcast_v16i64_to_v32i32_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2879,10 +2963,18 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2915,7 +3007,7 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2949,8 +3041,6 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v32i32_to_v16f64_scalar:
; VI: ; %bb.0:
@@ -2975,10 +3065,18 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -3011,7 +3109,7 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3045,8 +3143,6 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v32i32_to_v16f64_scalar:
; GFX9: ; %bb.0:
@@ -3071,10 +3167,18 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -3107,7 +3211,7 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3141,8 +3245,6 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v32i32_to_v16f64_scalar:
; GFX11: ; %bb.0:
@@ -3164,11 +3266,16 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -3201,7 +3308,7 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3220,8 +3327,6 @@ define inreg <16 x double> @bitcast_v32i32_to_v16f64_scalar(<32 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3452,10 +3557,18 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -3472,10 +3585,8 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -3508,7 +3619,7 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -3587,10 +3698,18 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -3607,10 +3726,8 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -3643,7 +3760,7 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -3722,10 +3839,18 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -3742,10 +3867,8 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -3778,7 +3901,7 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -3858,11 +3981,16 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -3880,8 +4008,6 @@ define inreg <32 x i32> @bitcast_v16f64_to_v32i32_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -7962,607 +8088,597 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s45, v2
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: ; implicit-def: $vgpr21 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; SI-NEXT: ; implicit-def: $vgpr21 : SGPR spill to VGPR lane
; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s46, s5, 24
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s46, 6
+; SI-NEXT: v_writelane_b32 v22, s46, 60
; SI-NEXT: s_lshr_b32 s46, s5, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 7
+; SI-NEXT: v_writelane_b32 v22, s46, 61
+; SI-NEXT: s_lshr_b32 s46, s13, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 62
; SI-NEXT: s_lshr_b32 s46, s15, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 8
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 2
-; SI-NEXT: v_writelane_b32 v22, s47, 3
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 4
-; SI-NEXT: v_writelane_b32 v22, s47, 5
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 0
-; SI-NEXT: v_writelane_b32 v22, s47, 1
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 62
-; SI-NEXT: v_writelane_b32 v21, s47, 63
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 60
-; SI-NEXT: v_writelane_b32 v21, s47, 61
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 58
-; SI-NEXT: v_writelane_b32 v21, s47, 59
+; SI-NEXT: v_writelane_b32 v22, s46, 63
+; SI-NEXT: s_lshr_b32 s46, s15, 16
+; SI-NEXT: v_writelane_b32 v21, s46, 0
+; SI-NEXT: s_lshr_b32 s46, s15, 8
+; SI-NEXT: v_writelane_b32 v21, s46, 1
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 56
-; SI-NEXT: v_writelane_b32 v21, s47, 57
+; SI-NEXT: v_writelane_b32 v22, s46, 58
+; SI-NEXT: v_writelane_b32 v22, s47, 59
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 54
-; SI-NEXT: v_writelane_b32 v21, s47, 55
+; SI-NEXT: v_writelane_b32 v22, s46, 56
+; SI-NEXT: v_writelane_b32 v22, s47, 57
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 52
-; SI-NEXT: v_writelane_b32 v21, s47, 53
+; SI-NEXT: v_writelane_b32 v22, s46, 54
+; SI-NEXT: v_writelane_b32 v22, s47, 55
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 50
-; SI-NEXT: v_writelane_b32 v21, s47, 51
+; SI-NEXT: v_writelane_b32 v22, s46, 52
+; SI-NEXT: v_writelane_b32 v22, s47, 53
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 48
-; SI-NEXT: v_writelane_b32 v21, s47, 49
+; SI-NEXT: v_writelane_b32 v22, s46, 50
+; SI-NEXT: v_writelane_b32 v22, s47, 51
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 46
-; SI-NEXT: v_writelane_b32 v21, s47, 47
+; SI-NEXT: v_writelane_b32 v22, s46, 48
+; SI-NEXT: v_writelane_b32 v22, s47, 49
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 44
-; SI-NEXT: v_writelane_b32 v21, s47, 45
+; SI-NEXT: v_writelane_b32 v22, s46, 46
+; SI-NEXT: v_writelane_b32 v22, s47, 47
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 42
-; SI-NEXT: v_writelane_b32 v21, s47, 43
+; SI-NEXT: v_writelane_b32 v22, s46, 44
+; SI-NEXT: v_writelane_b32 v22, s47, 45
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 40
-; SI-NEXT: v_writelane_b32 v21, s47, 41
+; SI-NEXT: v_writelane_b32 v22, s46, 42
+; SI-NEXT: v_writelane_b32 v22, s47, 43
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 38
-; SI-NEXT: v_writelane_b32 v21, s47, 39
+; SI-NEXT: v_writelane_b32 v22, s46, 40
+; SI-NEXT: v_writelane_b32 v22, s47, 41
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 36
-; SI-NEXT: v_writelane_b32 v21, s47, 37
+; SI-NEXT: v_writelane_b32 v22, s46, 38
+; SI-NEXT: v_writelane_b32 v22, s47, 39
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 34
-; SI-NEXT: v_writelane_b32 v21, s47, 35
+; SI-NEXT: v_writelane_b32 v22, s46, 36
+; SI-NEXT: v_writelane_b32 v22, s47, 37
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 32
-; SI-NEXT: v_writelane_b32 v21, s47, 33
+; SI-NEXT: v_writelane_b32 v22, s46, 34
+; SI-NEXT: v_writelane_b32 v22, s47, 35
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 30
-; SI-NEXT: v_writelane_b32 v21, s47, 31
+; SI-NEXT: v_writelane_b32 v22, s46, 32
+; SI-NEXT: v_writelane_b32 v22, s47, 33
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 28
-; SI-NEXT: v_writelane_b32 v21, s47, 29
-; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 26
-; SI-NEXT: v_writelane_b32 v21, s47, 27
-; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 24
-; SI-NEXT: v_writelane_b32 v21, s47, 25
+; SI-NEXT: v_writelane_b32 v22, s46, 30
+; SI-NEXT: v_writelane_b32 v22, s47, 31
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 22
-; SI-NEXT: v_writelane_b32 v21, s47, 23
+; SI-NEXT: v_writelane_b32 v22, s46, 28
+; SI-NEXT: v_writelane_b32 v22, s47, 29
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 20
-; SI-NEXT: v_writelane_b32 v21, s47, 21
+; SI-NEXT: v_writelane_b32 v22, s46, 26
+; SI-NEXT: v_writelane_b32 v22, s47, 27
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 18
-; SI-NEXT: v_writelane_b32 v21, s47, 19
+; SI-NEXT: v_writelane_b32 v22, s46, 24
+; SI-NEXT: v_writelane_b32 v22, s47, 25
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 16
-; SI-NEXT: v_writelane_b32 v21, s47, 17
+; SI-NEXT: v_writelane_b32 v22, s46, 22
+; SI-NEXT: v_writelane_b32 v22, s47, 23
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 14
-; SI-NEXT: v_writelane_b32 v21, s47, 15
+; SI-NEXT: v_writelane_b32 v22, s46, 20
+; SI-NEXT: v_writelane_b32 v22, s47, 21
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 8
-; SI-NEXT: v_writelane_b32 v21, s46, 12
-; SI-NEXT: v_writelane_b32 v21, s47, 13
+; SI-NEXT: v_writelane_b32 v22, s46, 18
+; SI-NEXT: v_writelane_b32 v22, s47, 19
; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 10
-; SI-NEXT: v_writelane_b32 v21, s47, 11
-; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 16
+; SI-NEXT: v_writelane_b32 v22, s47, 17
+; SI-NEXT: s_lshr_b64 s[58:59], s[26:27], 8
+; SI-NEXT: v_writelane_b32 v22, s58, 14
+; SI-NEXT: v_writelane_b32 v22, s59, 15
+; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v22, s58, 12
+; SI-NEXT: v_writelane_b32 v22, s59, 13
+; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 16
+; SI-NEXT: v_writelane_b32 v22, s58, 10
+; SI-NEXT: v_writelane_b32 v22, s59, 11
+; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 8
+; SI-NEXT: v_writelane_b32 v22, s58, 8
+; SI-NEXT: v_writelane_b32 v22, s59, 9
+; SI-NEXT: s_lshr_b64 s[58:59], s[22:23], 24
+; SI-NEXT: v_writelane_b32 v22, s58, 6
+; SI-NEXT: v_writelane_b32 v22, s59, 7
+; SI-NEXT: s_lshr_b64 s[58:59], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v22, s58, 4
+; SI-NEXT: v_writelane_b32 v22, s59, 5
+; SI-NEXT: s_lshr_b64 s[58:59], s[22:23], 8
+; SI-NEXT: s_lshr_b32 s72, s5, 8
; SI-NEXT: s_lshr_b32 s74, s7, 24
-; SI-NEXT: v_writelane_b32 v21, s47, 9
-; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 24
-; SI-NEXT: s_lshr_b32 s76, s7, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 6
-; SI-NEXT: s_mov_b32 s73, s74
-; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
-; SI-NEXT: s_lshr_b32 s78, s7, 8
-; SI-NEXT: v_writelane_b32 v21, s47, 7
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 24
-; SI-NEXT: s_mov_b32 s75, s76
-; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 8
-; SI-NEXT: s_lshr_b32 s88, s9, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 4
-; SI-NEXT: s_mov_b32 s77, s78
-; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 24
+; SI-NEXT: s_lshr_b32 s75, s45, 24
+; SI-NEXT: s_lshr_b64 s[84:85], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[66:67], s[42:43], 16
+; SI-NEXT: v_writelane_b32 v22, s58, 2
+; SI-NEXT: s_lshr_b32 s63, s43, 8
+; SI-NEXT: s_lshr_b32 s62, s21, 8
+; SI-NEXT: s_lshr_b64 s[64:65], s[42:43], 24
+; SI-NEXT: s_mov_b32 s85, s72
+; SI-NEXT: s_mov_b32 s67, s75
+; SI-NEXT: s_mov_b32 s72, s74
+; SI-NEXT: v_writelane_b32 v22, s59, 3
+; SI-NEXT: s_lshr_b64 s[74:75], s[20:21], 24
+; SI-NEXT: s_lshr_b64 s[58:59], s[20:21], 16
; SI-NEXT: s_lshr_b32 s90, s9, 16
-; SI-NEXT: v_writelane_b32 v21, s47, 5
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 16
-; SI-NEXT: s_mov_b32 s79, s88
-; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b32 s92, s9, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 2
-; SI-NEXT: s_mov_b32 s89, s90
-; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 8
+; SI-NEXT: s_lshr_b32 s93, s25, 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 24
+; SI-NEXT: s_mov_b32 s65, s63
+; SI-NEXT: s_mov_b32 s75, s62
+; SI-NEXT: v_writelane_b32 v22, s58, 0
+; SI-NEXT: s_lshr_b64 s[62:63], s[20:21], 8
+; SI-NEXT: s_lshr_b32 s76, s7, 16
; SI-NEXT: s_lshr_b32 s94, s11, 24
-; SI-NEXT: s_lshr_b32 s36, s13, 24
-; SI-NEXT: v_writelane_b32 v21, s47, 3
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 8
-; SI-NEXT: s_mov_b32 s91, s92
-; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
; SI-NEXT: s_lshr_b32 vcc_lo, s11, 16
; SI-NEXT: s_lshr_b32 vcc_hi, s11, 8
-; SI-NEXT: s_lshr_b32 s38, s13, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 0
+; SI-NEXT: s_lshr_b32 s77, s45, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 24
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[42:43], 8
+; SI-NEXT: v_writelane_b32 v22, s59, 1
+; SI-NEXT: s_mov_b32 s97, s90
+; SI-NEXT: s_mov_b32 s63, s93
+; SI-NEXT: s_mov_b32 s90, s92
+; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
+; SI-NEXT: s_lshr_b32 s78, s7, 8
+; SI-NEXT: s_lshr_b32 s88, s9, 24
+; SI-NEXT: s_lshr_b32 s61, s41, 24
+; SI-NEXT: s_lshr_b32 s89, s27, 24
+; SI-NEXT: s_lshr_b32 s34, s19, 8
+; SI-NEXT: s_lshr_b32 s35, s17, 24
+; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 8
+; SI-NEXT: s_lshr_b64 s[98:99], s[6:7], 8
+; SI-NEXT: s_mov_b32 s69, s77
+; SI-NEXT: s_mov_b32 s55, s76
; SI-NEXT: s_mov_b32 s93, s94
-; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 8
+; SI-NEXT: s_mov_b32 s59, s30
+; SI-NEXT: s_mov_b32 s94, vcc_lo
+; SI-NEXT: s_lshr_b64 s[76:77], s[18:19], 8
+; SI-NEXT: s_mov_b32 s30, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b32 s36, s13, 24
+; SI-NEXT: s_lshr_b32 s38, s13, 16
+; SI-NEXT: s_lshr_b32 s39, s17, 8
+; SI-NEXT: s_mov_b32 s99, s61
+; SI-NEXT: s_mov_b32 s87, s78
+; SI-NEXT: s_mov_b32 s61, s89
+; SI-NEXT: s_mov_b32 s78, s88
+; SI-NEXT: s_mov_b32 s77, s34
+; SI-NEXT: s_mov_b32 vcc_hi, s35
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 8
+; SI-NEXT: s_lshr_b32 s53, s41, 16
+; SI-NEXT: s_lshr_b32 s49, s41, 8
+; SI-NEXT: s_lshr_b32 s73, s43, 24
+; SI-NEXT: s_lshr_b32 s48, s43, 16
+; SI-NEXT: s_lshr_b32 s50, s45, 8
+; SI-NEXT: s_lshr_b32 s51, s29, 24
+; SI-NEXT: s_lshr_b32 s70, s29, 16
+; SI-NEXT: s_lshr_b32 s79, s29, 8
+; SI-NEXT: s_lshr_b32 s71, s27, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 8
+; SI-NEXT: s_lshr_b32 s91, s25, 24
+; SI-NEXT: s_lshr_b32 s81, s25, 8
+; SI-NEXT: s_lshr_b32 s82, s23, 24
+; SI-NEXT: s_lshr_b32 s83, s23, 16
+; SI-NEXT: s_lshr_b32 s95, s23, 8
+; SI-NEXT: s_lshr_b32 s60, s21, 16
+; SI-NEXT: s_lshr_b32 s52, s19, 24
+; SI-NEXT: s_lshr_b32 s31, s19, 16
+; SI-NEXT: s_lshr_b32 s37, s17, 16
+; SI-NEXT: s_lshr_b64 s[56:57], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
; SI-NEXT: s_mov_b32 s35, s36
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_lshr_b32 s63, s5, 8
-; SI-NEXT: s_lshr_b32 s39, s13, 8
-; SI-NEXT: s_lshr_b32 s87, s15, 16
-; SI-NEXT: s_lshr_b32 s55, s15, 8
-; SI-NEXT: s_lshr_b32 s97, s41, 24
-; SI-NEXT: s_lshr_b32 s54, s41, 16
-; SI-NEXT: s_lshr_b32 s99, s41, 8
-; SI-NEXT: s_lshr_b32 s85, s43, 24
-; SI-NEXT: s_lshr_b32 s98, s43, 16
-; SI-NEXT: s_lshr_b32 s53, s43, 8
-; SI-NEXT: s_lshr_b32 s96, s45, 24
-; SI-NEXT: s_lshr_b32 s86, s45, 16
-; SI-NEXT: s_lshr_b32 s49, s45, 8
-; SI-NEXT: s_lshr_b32 s64, s29, 24
-; SI-NEXT: s_lshr_b32 s65, s29, 16
-; SI-NEXT: s_lshr_b32 s66, s29, 8
-; SI-NEXT: s_lshr_b32 s67, s27, 24
-; SI-NEXT: s_lshr_b32 s58, s27, 16
-; SI-NEXT: s_lshr_b32 s59, s27, 8
-; SI-NEXT: s_lshr_b32 s60, s25, 24
-; SI-NEXT: s_lshr_b32 s61, s25, 16
-; SI-NEXT: s_lshr_b32 s62, s25, 8
-; SI-NEXT: s_lshr_b32 s68, s23, 24
-; SI-NEXT: s_lshr_b32 s69, s23, 16
-; SI-NEXT: s_lshr_b32 s48, s23, 8
-; SI-NEXT: s_lshr_b32 s50, s21, 24
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 8
-; SI-NEXT: s_lshr_b32 s71, s19, 24
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s19, 8
-; SI-NEXT: s_lshr_b32 s82, s17, 24
-; SI-NEXT: s_lshr_b32 s83, s17, 16
-; SI-NEXT: s_lshr_b32 s52, s17, 8
-; SI-NEXT: s_lshr_b64 s[56:57], s[42:43], 24
-; SI-NEXT: v_writelane_b32 v21, s47, 1
-; SI-NEXT: s_mov_b32 s95, vcc_lo
-; SI-NEXT: s_mov_b32 s31, vcc_hi
-; SI-NEXT: s_mov_b32 s37, s38
-; SI-NEXT: s_lshr_b64 s[46:47], s[16:17], 8
-; SI-NEXT: s_mov_b64 vcc, 0
+; SI-NEXT: s_mov_b32 s89, s39
+; SI-NEXT: s_mov_b32 s36, s38
+; SI-NEXT: s_mov_b64 s[38:39], 0
; SI-NEXT: s_branch .LBB13_3
; SI-NEXT: .LBB13_2:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr84
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: s_mov_b64 vcc, -1
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v21, s56, 0
-; SI-NEXT: v_writelane_b32 v21, s57, 1
; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: s_mov_b64 s[38:39], -1
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v21, s56, 2
-; SI-NEXT: v_writelane_b32 v21, s57, 3
-; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr83
; SI-NEXT: ; implicit-def: $sgpr82
; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr91
; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr79
; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr51
; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr48
; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr73
; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr96
; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr35
; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr97
; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: v_writelane_b32 v21, s56, 4
-; SI-NEXT: v_writelane_b32 v21, s57, 5
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v22, s46, 0
+; SI-NEXT: v_writelane_b32 v22, s47, 1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 2
+; SI-NEXT: v_writelane_b32 v22, s47, 3
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 4
+; SI-NEXT: v_writelane_b32 v22, s47, 5
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 6
+; SI-NEXT: v_writelane_b32 v22, s47, 7
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 8
+; SI-NEXT: v_writelane_b32 v22, s47, 9
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 10
+; SI-NEXT: v_writelane_b32 v22, s47, 11
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 12
+; SI-NEXT: v_writelane_b32 v22, s47, 13
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s46, 14
+; SI-NEXT: v_writelane_b32 v22, s47, 15
+; SI-NEXT: v_writelane_b32 v22, s56, 16
+; SI-NEXT: v_writelane_b32 v22, s57, 17
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 6
-; SI-NEXT: v_writelane_b32 v21, s57, 7
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v22, s56, 18
+; SI-NEXT: v_writelane_b32 v22, s57, 19
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 8
-; SI-NEXT: v_writelane_b32 v21, s57, 9
+; SI-NEXT: v_writelane_b32 v22, s56, 20
+; SI-NEXT: v_writelane_b32 v22, s57, 21
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 10
-; SI-NEXT: v_writelane_b32 v21, s57, 11
+; SI-NEXT: v_writelane_b32 v22, s56, 22
+; SI-NEXT: v_writelane_b32 v22, s57, 23
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 12
-; SI-NEXT: v_writelane_b32 v21, s57, 13
+; SI-NEXT: v_writelane_b32 v22, s56, 24
+; SI-NEXT: v_writelane_b32 v22, s57, 25
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 14
-; SI-NEXT: v_writelane_b32 v21, s57, 15
+; SI-NEXT: v_writelane_b32 v22, s56, 26
+; SI-NEXT: v_writelane_b32 v22, s57, 27
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 16
-; SI-NEXT: v_writelane_b32 v21, s57, 17
+; SI-NEXT: v_writelane_b32 v22, s56, 28
+; SI-NEXT: v_writelane_b32 v22, s57, 29
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 18
-; SI-NEXT: v_writelane_b32 v21, s57, 19
+; SI-NEXT: v_writelane_b32 v22, s56, 30
+; SI-NEXT: v_writelane_b32 v22, s57, 31
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 20
-; SI-NEXT: v_writelane_b32 v21, s57, 21
+; SI-NEXT: v_writelane_b32 v22, s56, 32
+; SI-NEXT: v_writelane_b32 v22, s57, 33
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 22
-; SI-NEXT: v_writelane_b32 v21, s57, 23
+; SI-NEXT: v_writelane_b32 v22, s56, 34
+; SI-NEXT: v_writelane_b32 v22, s57, 35
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 24
-; SI-NEXT: v_writelane_b32 v21, s57, 25
+; SI-NEXT: v_writelane_b32 v22, s56, 36
+; SI-NEXT: v_writelane_b32 v22, s57, 37
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s56, 26
-; SI-NEXT: v_writelane_b32 v21, s57, 27
-; SI-NEXT: v_writelane_b32 v21, s38, 28
-; SI-NEXT: v_writelane_b32 v21, s39, 29
-; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: v_writelane_b32 v22, s56, 38
+; SI-NEXT: v_writelane_b32 v22, s57, 39
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 40
+; SI-NEXT: v_writelane_b32 v22, s57, 41
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 42
+; SI-NEXT: v_writelane_b32 v22, s57, 43
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 44
+; SI-NEXT: v_writelane_b32 v22, s57, 45
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 46
+; SI-NEXT: v_writelane_b32 v22, s57, 47
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 48
+; SI-NEXT: v_writelane_b32 v22, s57, 49
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 50
+; SI-NEXT: v_writelane_b32 v22, s57, 51
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 52
+; SI-NEXT: v_writelane_b32 v22, s57, 53
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 54
+; SI-NEXT: v_writelane_b32 v22, s57, 55
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 56
+; SI-NEXT: v_writelane_b32 v22, s57, 57
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: v_writelane_b32 v22, s56, 58
+; SI-NEXT: v_writelane_b32 v22, s57, 59
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: v_writelane_b32 v21, s38, 30
-; SI-NEXT: v_writelane_b32 v21, s39, 31
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: v_writelane_b32 v21, s38, 32
-; SI-NEXT: v_writelane_b32 v21, s39, 33
-; SI-NEXT: v_writelane_b32 v21, s84, 34
-; SI-NEXT: v_writelane_b32 v21, s85, 35
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: v_writelane_b32 v21, s84, 36
-; SI-NEXT: v_writelane_b32 v21, s85, 37
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 38
-; SI-NEXT: v_writelane_b32 v21, s85, 39
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 40
-; SI-NEXT: v_writelane_b32 v21, s85, 41
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 42
-; SI-NEXT: v_writelane_b32 v21, s85, 43
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 44
-; SI-NEXT: v_writelane_b32 v21, s85, 45
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 46
-; SI-NEXT: v_writelane_b32 v21, s85, 47
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 48
-; SI-NEXT: v_writelane_b32 v21, s85, 49
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 50
-; SI-NEXT: v_writelane_b32 v21, s85, 51
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 52
-; SI-NEXT: v_writelane_b32 v21, s85, 53
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 54
-; SI-NEXT: v_writelane_b32 v21, s85, 55
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 56
-; SI-NEXT: v_writelane_b32 v21, s85, 57
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 58
-; SI-NEXT: v_writelane_b32 v21, s85, 59
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 60
-; SI-NEXT: v_writelane_b32 v21, s85, 61
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v21, s84, 62
-; SI-NEXT: v_writelane_b32 v21, s85, 63
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s84, 0
-; SI-NEXT: v_writelane_b32 v22, s85, 1
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: v_writelane_b32 v22, s84, 2
-; SI-NEXT: v_writelane_b32 v22, s85, 3
-; SI-NEXT: v_writelane_b32 v22, s38, 4
-; SI-NEXT: v_writelane_b32 v22, s39, 5
; SI-NEXT: .LBB13_3: ; %Flow
-; SI-NEXT: s_andn2_b64 vcc, exec, vcc
-; SI-NEXT: s_mov_b32 s38, s98
-; SI-NEXT: s_mov_b32 s98, s85
-; SI-NEXT: s_mov_b32 s84, s56
+; SI-NEXT: s_and_b64 s[38:39], s[38:39], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_mov_b32 s38, s56
; SI-NEXT: s_mov_b32 s56, s53
-; SI-NEXT: s_mov_b32 s85, s86
-; SI-NEXT: s_mov_b32 s86, s49
-; SI-NEXT: s_mov_b32 s49, s64
-; SI-NEXT: s_mov_b32 s64, s65
-; SI-NEXT: s_mov_b32 s65, s66
-; SI-NEXT: s_mov_b32 s66, s67
-; SI-NEXT: s_mov_b32 s67, s58
-; SI-NEXT: s_cbranch_vccnz .LBB13_5
+; SI-NEXT: s_mov_b32 s39, s49
+; SI-NEXT: s_mov_b32 s49, s73
+; SI-NEXT: s_mov_b32 s53, s48
+; SI-NEXT: s_mov_b32 s73, s50
+; SI-NEXT: s_mov_b32 s48, s51
+; SI-NEXT: s_mov_b32 s50, s70
+; SI-NEXT: s_mov_b32 s51, s79
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s5, s5, 3
-; SI-NEXT: s_add_i32 s4, s4, 3
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 2
-; SI-NEXT: v_writelane_b32 v22, s47, 3
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 4
-; SI-NEXT: v_writelane_b32 v22, s47, 5
-; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 8
-; SI-NEXT: s_add_i32 s7, s7, 3
-; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_writelane_b32 v22, s46, 0
-; SI-NEXT: v_writelane_b32 v22, s47, 1
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 62
-; SI-NEXT: v_writelane_b32 v21, s47, 63
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 60
-; SI-NEXT: v_writelane_b32 v21, s47, 61
-; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 8
; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_add_i32 s8, s8, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 58
-; SI-NEXT: v_writelane_b32 v21, s47, 59
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 56
-; SI-NEXT: v_writelane_b32 v21, s47, 57
+; SI-NEXT: v_writelane_b32 v22, s46, 58
+; SI-NEXT: v_writelane_b32 v22, s47, 59
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 54
-; SI-NEXT: v_writelane_b32 v21, s47, 55
+; SI-NEXT: v_writelane_b32 v22, s46, 56
+; SI-NEXT: v_writelane_b32 v22, s47, 57
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 8
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_add_i32 s10, s10, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 52
-; SI-NEXT: v_writelane_b32 v21, s47, 53
+; SI-NEXT: v_writelane_b32 v22, s46, 54
+; SI-NEXT: v_writelane_b32 v22, s47, 55
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 50
-; SI-NEXT: v_writelane_b32 v21, s47, 51
+; SI-NEXT: v_writelane_b32 v22, s46, 52
+; SI-NEXT: v_writelane_b32 v22, s47, 53
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 48
-; SI-NEXT: v_writelane_b32 v21, s47, 49
+; SI-NEXT: v_writelane_b32 v22, s46, 50
+; SI-NEXT: v_writelane_b32 v22, s47, 51
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 8
; SI-NEXT: s_add_i32 s13, s13, 3
; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 46
-; SI-NEXT: v_writelane_b32 v21, s47, 47
+; SI-NEXT: v_writelane_b32 v22, s46, 48
+; SI-NEXT: v_writelane_b32 v22, s47, 49
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 44
-; SI-NEXT: v_writelane_b32 v21, s47, 45
+; SI-NEXT: v_writelane_b32 v22, s46, 46
+; SI-NEXT: v_writelane_b32 v22, s47, 47
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 42
-; SI-NEXT: v_writelane_b32 v21, s47, 43
+; SI-NEXT: v_writelane_b32 v22, s46, 44
+; SI-NEXT: v_writelane_b32 v22, s47, 45
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 8
; SI-NEXT: s_add_i32 s15, s15, 3
; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 40
-; SI-NEXT: v_writelane_b32 v21, s47, 41
+; SI-NEXT: v_writelane_b32 v22, s46, 42
+; SI-NEXT: v_writelane_b32 v22, s47, 43
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 38
-; SI-NEXT: v_writelane_b32 v21, s47, 39
+; SI-NEXT: v_writelane_b32 v22, s46, 40
+; SI-NEXT: v_writelane_b32 v22, s47, 41
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 36
-; SI-NEXT: v_writelane_b32 v21, s47, 37
+; SI-NEXT: v_writelane_b32 v22, s46, 38
+; SI-NEXT: v_writelane_b32 v22, s47, 39
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 8
; SI-NEXT: s_add_i32 s41, s41, 3
; SI-NEXT: s_add_i32 s40, s40, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 34
-; SI-NEXT: v_writelane_b32 v21, s47, 35
+; SI-NEXT: v_writelane_b32 v22, s46, 36
+; SI-NEXT: v_writelane_b32 v22, s47, 37
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 32
-; SI-NEXT: v_writelane_b32 v21, s47, 33
+; SI-NEXT: v_writelane_b32 v22, s46, 34
+; SI-NEXT: v_writelane_b32 v22, s47, 35
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 30
-; SI-NEXT: v_writelane_b32 v21, s47, 31
+; SI-NEXT: v_writelane_b32 v22, s46, 32
+; SI-NEXT: v_writelane_b32 v22, s47, 33
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 8
-; SI-NEXT: s_add_i32 s43, s43, 3
-; SI-NEXT: s_add_i32 s42, s42, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 28
-; SI-NEXT: v_writelane_b32 v21, s47, 29
-; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 26
-; SI-NEXT: v_writelane_b32 v21, s47, 27
-; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 8
; SI-NEXT: s_add_i32 s45, s45, 3
; SI-NEXT: s_add_i32 s44, s44, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 24
-; SI-NEXT: v_writelane_b32 v21, s47, 25
+; SI-NEXT: v_writelane_b32 v22, s46, 30
+; SI-NEXT: v_writelane_b32 v22, s47, 31
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 22
-; SI-NEXT: v_writelane_b32 v21, s47, 23
+; SI-NEXT: v_writelane_b32 v22, s46, 28
+; SI-NEXT: v_writelane_b32 v22, s47, 29
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 20
-; SI-NEXT: v_writelane_b32 v21, s47, 21
+; SI-NEXT: v_writelane_b32 v22, s46, 26
+; SI-NEXT: v_writelane_b32 v22, s47, 27
; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 8
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 18
-; SI-NEXT: v_writelane_b32 v21, s47, 19
+; SI-NEXT: v_writelane_b32 v22, s46, 24
+; SI-NEXT: v_writelane_b32 v22, s47, 25
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 16
-; SI-NEXT: v_writelane_b32 v21, s47, 17
+; SI-NEXT: v_writelane_b32 v22, s46, 22
+; SI-NEXT: v_writelane_b32 v22, s47, 23
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 14
-; SI-NEXT: v_writelane_b32 v21, s47, 15
+; SI-NEXT: v_writelane_b32 v22, s46, 20
+; SI-NEXT: v_writelane_b32 v22, s47, 21
; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 8
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 12
-; SI-NEXT: v_writelane_b32 v21, s47, 13
+; SI-NEXT: v_writelane_b32 v22, s46, 18
+; SI-NEXT: v_writelane_b32 v22, s47, 19
; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 10
-; SI-NEXT: v_writelane_b32 v21, s47, 11
-; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 8
-; SI-NEXT: v_writelane_b32 v21, s47, 9
-; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 8
+; SI-NEXT: v_writelane_b32 v22, s46, 16
+; SI-NEXT: v_writelane_b32 v22, s47, 17
+; SI-NEXT: s_lshr_b64 s[56:57], s[26:27], 8
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 6
-; SI-NEXT: v_writelane_b32 v21, s47, 7
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 24
-; SI-NEXT: v_writelane_b32 v21, s46, 4
-; SI-NEXT: v_writelane_b32 v21, s47, 5
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 16
-; SI-NEXT: v_writelane_b32 v21, s46, 2
-; SI-NEXT: v_writelane_b32 v21, s47, 3
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 8
+; SI-NEXT: v_writelane_b32 v22, s56, 14
+; SI-NEXT: v_writelane_b32 v22, s57, 15
+; SI-NEXT: s_lshr_b64 s[56:57], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v22, s56, 12
+; SI-NEXT: v_writelane_b32 v22, s57, 13
+; SI-NEXT: s_lshr_b64 s[56:57], s[24:25], 16
+; SI-NEXT: v_writelane_b32 v22, s56, 10
+; SI-NEXT: v_writelane_b32 v22, s57, 11
+; SI-NEXT: s_lshr_b64 s[56:57], s[24:25], 8
+; SI-NEXT: s_add_i32 s23, s23, 3
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: v_writelane_b32 v22, s56, 8
+; SI-NEXT: v_writelane_b32 v22, s57, 9
+; SI-NEXT: s_lshr_b64 s[56:57], s[22:23], 24
+; SI-NEXT: v_writelane_b32 v22, s56, 6
+; SI-NEXT: v_writelane_b32 v22, s57, 7
+; SI-NEXT: s_lshr_b64 s[56:57], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v22, s56, 4
+; SI-NEXT: v_writelane_b32 v22, s57, 5
+; SI-NEXT: s_lshr_b64 s[56:57], s[22:23], 8
+; SI-NEXT: s_add_i32 s21, s21, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: v_writelane_b32 v22, s56, 2
+; SI-NEXT: v_writelane_b32 v22, s57, 3
+; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
+; SI-NEXT: s_add_i32 s5, s5, 3
+; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
+; SI-NEXT: v_writelane_b32 v22, s56, 0
+; SI-NEXT: v_writelane_b32 v22, s57, 1
+; SI-NEXT: s_lshr_b32 s47, s5, 24
+; SI-NEXT: v_writelane_b32 v22, s47, 60
+; SI-NEXT: s_lshr_b32 s47, s5, 16
+; SI-NEXT: v_writelane_b32 v22, s47, 61
+; SI-NEXT: s_lshr_b32 s47, s13, 8
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_writelane_b32 v21, s46, 0
-; SI-NEXT: v_writelane_b32 v21, s47, 1
-; SI-NEXT: s_lshr_b64 s[46:47], s[16:17], 8
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_lshr_b32 s47, s5, 24
-; SI-NEXT: s_lshr_b64 s[84:85], s[42:43], 24
-; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 8
-; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 8
+; SI-NEXT: s_add_i32 s43, s43, 3
+; SI-NEXT: s_add_i32 s42, s42, 3
+; SI-NEXT: s_add_i32 s7, s7, 3
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: s_add_i32 s4, s4, 3
+; SI-NEXT: v_writelane_b32 v22, s47, 62
+; SI-NEXT: s_lshr_b32 s47, s15, 24
+; SI-NEXT: s_lshr_b64 s[38:39], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[84:85], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 8
+; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 24
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[98:99], s[6:7], 8
+; SI-NEXT: s_lshr_b64 s[64:65], s[42:43], 24
+; SI-NEXT: s_lshr_b64 s[66:67], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[42:43], 8
+; SI-NEXT: s_lshr_b64 s[74:75], s[20:21], 24
+; SI-NEXT: s_lshr_b64 s[62:63], s[20:21], 8
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: v_writelane_b32 v22, s47, 6
-; SI-NEXT: s_lshr_b32 s47, s5, 16
-; SI-NEXT: v_writelane_b32 v22, s47, 7
-; SI-NEXT: s_lshr_b32 s63, s5, 8
-; SI-NEXT: s_lshr_b32 s73, s7, 24
-; SI-NEXT: s_lshr_b32 s75, s7, 16
-; SI-NEXT: s_lshr_b32 s77, s7, 8
-; SI-NEXT: s_lshr_b32 s79, s9, 24
-; SI-NEXT: s_lshr_b32 s89, s9, 16
-; SI-NEXT: s_lshr_b32 s91, s9, 8
+; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[18:19], 8
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 8
+; SI-NEXT: v_writelane_b32 v22, s47, 63
+; SI-NEXT: s_lshr_b32 s47, s15, 16
+; SI-NEXT: s_lshr_b32 s85, s5, 8
+; SI-NEXT: s_lshr_b32 s72, s7, 24
+; SI-NEXT: s_lshr_b32 s55, s7, 16
+; SI-NEXT: s_lshr_b32 s87, s7, 8
+; SI-NEXT: s_lshr_b32 s78, s9, 24
+; SI-NEXT: s_lshr_b32 s97, s9, 16
+; SI-NEXT: s_lshr_b32 s90, s9, 8
; SI-NEXT: s_lshr_b32 s93, s11, 24
-; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s31, s11, 8
+; SI-NEXT: s_lshr_b32 s94, s11, 16
+; SI-NEXT: s_lshr_b32 s30, s11, 8
; SI-NEXT: s_lshr_b32 s35, s13, 24
-; SI-NEXT: s_lshr_b32 s37, s13, 16
-; SI-NEXT: s_lshr_b32 s39, s13, 8
-; SI-NEXT: s_lshr_b32 s47, s15, 24
-; SI-NEXT: s_lshr_b32 s87, s15, 16
-; SI-NEXT: s_lshr_b32 s55, s15, 8
-; SI-NEXT: s_lshr_b32 s97, s41, 24
-; SI-NEXT: s_lshr_b32 s54, s41, 16
-; SI-NEXT: s_lshr_b32 s99, s41, 8
-; SI-NEXT: s_lshr_b32 s98, s43, 24
-; SI-NEXT: s_lshr_b32 s38, s43, 16
-; SI-NEXT: s_lshr_b32 s56, s43, 8
-; SI-NEXT: s_lshr_b32 s96, s45, 24
-; SI-NEXT: s_lshr_b32 s85, s45, 16
-; SI-NEXT: s_lshr_b32 s86, s45, 8
-; SI-NEXT: s_lshr_b32 s49, s29, 24
-; SI-NEXT: s_lshr_b32 s64, s29, 16
-; SI-NEXT: s_lshr_b32 s65, s29, 8
-; SI-NEXT: s_lshr_b32 s66, s27, 24
-; SI-NEXT: s_lshr_b32 s67, s27, 16
-; SI-NEXT: s_lshr_b32 s59, s27, 8
-; SI-NEXT: s_lshr_b32 s60, s25, 24
-; SI-NEXT: s_lshr_b32 s61, s25, 16
-; SI-NEXT: s_lshr_b32 s62, s25, 8
-; SI-NEXT: s_lshr_b32 s68, s23, 24
-; SI-NEXT: s_lshr_b32 s69, s23, 16
-; SI-NEXT: s_lshr_b32 s48, s23, 8
-; SI-NEXT: s_lshr_b32 s50, s21, 24
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 8
-; SI-NEXT: s_lshr_b32 s71, s19, 24
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s19, 8
-; SI-NEXT: s_lshr_b32 s82, s17, 24
-; SI-NEXT: s_lshr_b32 s83, s17, 16
-; SI-NEXT: s_lshr_b32 s52, s17, 8
-; SI-NEXT: v_writelane_b32 v22, s47, 8
+; SI-NEXT: s_lshr_b32 s36, s13, 16
+; SI-NEXT: v_writelane_b32 v21, s47, 0
+; SI-NEXT: s_lshr_b32 s47, s15, 8
+; SI-NEXT: s_lshr_b32 s99, s41, 24
+; SI-NEXT: s_lshr_b32 s56, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s41, 8
+; SI-NEXT: s_lshr_b32 s49, s43, 24
+; SI-NEXT: s_lshr_b32 s53, s43, 16
+; SI-NEXT: s_lshr_b32 s65, s43, 8
+; SI-NEXT: s_lshr_b32 s67, s45, 24
+; SI-NEXT: s_lshr_b32 s69, s45, 16
+; SI-NEXT: s_lshr_b32 s73, s45, 8
+; SI-NEXT: s_lshr_b32 s48, s29, 24
+; SI-NEXT: s_lshr_b32 s50, s29, 16
+; SI-NEXT: s_lshr_b32 s51, s29, 8
+; SI-NEXT: s_lshr_b32 s61, s27, 24
+; SI-NEXT: s_lshr_b32 s71, s27, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 8
+; SI-NEXT: s_lshr_b32 s91, s25, 24
+; SI-NEXT: s_lshr_b32 s63, s25, 16
+; SI-NEXT: s_lshr_b32 s81, s25, 8
+; SI-NEXT: s_lshr_b32 s82, s23, 24
+; SI-NEXT: s_lshr_b32 s83, s23, 16
+; SI-NEXT: s_lshr_b32 s95, s23, 8
+; SI-NEXT: s_lshr_b32 s59, s21, 24
+; SI-NEXT: s_lshr_b32 s60, s21, 16
+; SI-NEXT: s_lshr_b32 s75, s21, 8
+; SI-NEXT: s_lshr_b32 s52, s19, 24
+; SI-NEXT: s_lshr_b32 s31, s19, 16
+; SI-NEXT: s_lshr_b32 s77, s19, 8
+; SI-NEXT: s_lshr_b32 vcc_hi, s17, 24
+; SI-NEXT: s_lshr_b32 s37, s17, 16
+; SI-NEXT: s_lshr_b32 s89, s17, 8
+; SI-NEXT: v_writelane_b32 v21, s47, 1
; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s47, s46, 8
+; SI-NEXT: s_lshl_b32 s47, s88, 8
; SI-NEXT: s_or_b32 s16, s16, s47
-; SI-NEXT: s_and_b32 s47, s36, 0xff
+; SI-NEXT: s_and_b32 s47, s34, 0xff
; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_lshl_b32 s57, s34, 24
+; SI-NEXT: s_lshl_b32 s57, vcc_lo, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s47, s57, s47
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s17, 0xff
-; SI-NEXT: s_lshl_b32 s17, s52, 8
+; SI-NEXT: s_lshl_b32 s17, s89, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s83, 0xff
+; SI-NEXT: s_and_b32 s17, s37, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s47, s82, 24
+; SI-NEXT: s_lshl_b32 s47, vcc_hi, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s47, s17
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s18, 0xff
-; SI-NEXT: s_lshl_b32 s17, s30, 8
+; SI-NEXT: s_lshl_b32 s17, s76, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s94, 0xff
+; SI-NEXT: s_and_b32 s17, s58, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s92, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8575,11 +8691,11 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s19, 0xff
-; SI-NEXT: s_lshl_b32 s17, s81, 8
+; SI-NEXT: s_lshl_b32 s17, s77, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s80, 0xff
+; SI-NEXT: s_and_b32 s17, s31, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s71, 24
+; SI-NEXT: s_lshl_b32 s18, s52, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 8, v0
@@ -8588,11 +8704,12 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s20, 0xff
-; SI-NEXT: s_lshl_b32 s17, s90, 8
+; SI-NEXT: s_lshl_b32 s17, s62, 8
+; SI-NEXT: v_readlane_b32 s18, v22, 0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s88, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s78, 24
+; SI-NEXT: s_lshl_b32 s18, s74, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 12, v0
@@ -8601,24 +8718,30 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s21, 0xff
-; SI-NEXT: s_lshl_b32 s17, s70, 8
+; SI-NEXT: s_lshl_b32 s17, s75, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s51, 0xff
+; SI-NEXT: s_and_b32 s17, s60, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 1
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s50, 24
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s59, 24
; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v22, 2
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s19, v22, 3
; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_lshl_b32 s17, s18, 8
+; SI-NEXT: v_readlane_b32 s18, v22, 4
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s22, 0xff
-; SI-NEXT: s_lshl_b32 s17, s76, 8
+; SI-NEXT: v_readlane_b32 s19, v22, 5
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s74, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 6
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s72, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 20, v0
@@ -8627,27 +8750,28 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s23, 0xff
-; SI-NEXT: s_lshl_b32 s17, s48, 8
+; SI-NEXT: s_lshl_b32 s17, s95, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s69, 0xff
+; SI-NEXT: s_and_b32 s17, s83, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 7
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s68, 24
+; SI-NEXT: s_lshl_b32 s18, s82, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 0
+; SI-NEXT: v_readlane_b32 s18, v22, 8
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 1
+; SI-NEXT: v_readlane_b32 s19, v22, 9
; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 2
+; SI-NEXT: v_readlane_b32 s18, v22, 10
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s24, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 3
+; SI-NEXT: v_readlane_b32 s19, v22, 11
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 4
+; SI-NEXT: v_readlane_b32 s18, v22, 12
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8658,28 +8782,26 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s25, 0xff
-; SI-NEXT: s_lshl_b32 s17, s62, 8
+; SI-NEXT: s_lshl_b32 s17, s81, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s61, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 5
+; SI-NEXT: s_and_b32 s17, s63, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s60, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 6
+; SI-NEXT: s_lshl_b32 s18, s91, 24
+; SI-NEXT: v_readlane_b32 s19, v22, 13
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 7
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 8
+; SI-NEXT: v_readlane_b32 s18, v22, 14
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s26, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 9
+; SI-NEXT: v_readlane_b32 s19, v22, 15
+; SI-NEXT: s_lshl_b32 s17, s18, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 10
+; SI-NEXT: s_and_b32 s17, s46, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 16
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8690,28 +8812,28 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s27, 0xff
-; SI-NEXT: s_lshl_b32 s17, s59, 8
+; SI-NEXT: s_lshl_b32 s17, s80, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s67, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 11
+; SI-NEXT: s_and_b32 s17, s71, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 17
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s66, 24
+; SI-NEXT: s_lshl_b32 s18, s61, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 12
+; SI-NEXT: v_readlane_b32 s18, v22, 18
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 13
+; SI-NEXT: v_readlane_b32 s19, v22, 19
; SI-NEXT: v_add_i32_e32 v1, vcc, 40, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 14
+; SI-NEXT: v_readlane_b32 s18, v22, 20
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s28, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 15
+; SI-NEXT: v_readlane_b32 s19, v22, 21
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 16
+; SI-NEXT: v_readlane_b32 s18, v22, 22
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8722,28 +8844,28 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s29, 0xff
-; SI-NEXT: s_lshl_b32 s17, s65, 8
+; SI-NEXT: s_lshl_b32 s17, s51, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s64, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 17
+; SI-NEXT: s_and_b32 s17, s50, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 23
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s49, 24
+; SI-NEXT: s_lshl_b32 s18, s48, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 18
+; SI-NEXT: v_readlane_b32 s18, v22, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 19
+; SI-NEXT: v_readlane_b32 s19, v22, 25
; SI-NEXT: v_add_i32_e32 v1, vcc, 48, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 20
+; SI-NEXT: v_readlane_b32 s18, v22, 26
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 21
+; SI-NEXT: v_readlane_b32 s19, v22, 27
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 22
+; SI-NEXT: v_readlane_b32 s18, v22, 28
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8754,28 +8876,24 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s45, 0xff
-; SI-NEXT: s_lshl_b32 s17, s86, 8
+; SI-NEXT: s_lshl_b32 s17, s73, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s85, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 23
+; SI-NEXT: s_and_b32 s17, s69, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s96, 24
+; SI-NEXT: s_lshl_b32 s18, s67, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 24
; SI-NEXT: v_add_i32_e32 v1, vcc, 56, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s19, v21, 25
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 26
+; SI-NEXT: s_lshl_b32 s17, s68, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: s_and_b32 s17, s66, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s84, 24
+; SI-NEXT: s_lshl_b32 s18, s64, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 60, v0
@@ -8784,28 +8902,28 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s43, 0xff
-; SI-NEXT: s_lshl_b32 s17, s56, 8
+; SI-NEXT: s_lshl_b32 s17, s65, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s38, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 27
+; SI-NEXT: s_and_b32 s17, s53, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 29
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s98, 24
+; SI-NEXT: s_lshl_b32 s18, s49, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 28
+; SI-NEXT: v_readlane_b32 s18, v22, 30
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 29
+; SI-NEXT: v_readlane_b32 s19, v22, 31
; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 30
+; SI-NEXT: v_readlane_b32 s18, v22, 32
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 31
+; SI-NEXT: v_readlane_b32 s19, v22, 33
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 32
+; SI-NEXT: v_readlane_b32 s18, v22, 34
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8816,11 +8934,11 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s41, 0xff
-; SI-NEXT: s_lshl_b32 s17, s99, 8
+; SI-NEXT: s_lshl_b32 s17, s39, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s54, 0xff
+; SI-NEXT: s_and_b32 s17, s56, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s97, 24
+; SI-NEXT: s_lshl_b32 s18, s99, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x48, v0
@@ -8828,16 +8946,16 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
-; SI-NEXT: v_readlane_b32 s16, v21, 34
+; SI-NEXT: v_readlane_b32 s16, v22, 36
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v21, 35
+; SI-NEXT: v_readlane_b32 s17, v22, 37
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v21, 33
+; SI-NEXT: v_readlane_b32 s19, v22, 35
; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: v_readlane_b32 s16, v21, 36
-; SI-NEXT: v_readlane_b32 s17, v21, 37
+; SI-NEXT: v_readlane_b32 s16, v22, 38
+; SI-NEXT: v_readlane_b32 s17, v22, 39
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 38
+; SI-NEXT: v_readlane_b32 s18, v22, 40
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_lshl_b32 s17, s18, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
@@ -8848,10 +8966,12 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: s_and_b32 s14, s15, 0xff
-; SI-NEXT: s_lshl_b32 s15, s55, 8
+; SI-NEXT: v_readlane_b32 s15, v21, 1
+; SI-NEXT: s_lshl_b32 s15, s15, 8
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: s_and_b32 s15, s87, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 8
+; SI-NEXT: v_readlane_b32 s15, v21, 0
+; SI-NEXT: s_and_b32 s15, s15, 0xff
+; SI-NEXT: v_readlane_b32 s16, v22, 63
; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: s_lshl_b32 s16, s16, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
@@ -8861,15 +8981,15 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_readlane_b32 s14, v21, 40
+; SI-NEXT: v_readlane_b32 s14, v22, 42
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v21, 41
+; SI-NEXT: v_readlane_b32 s15, v22, 43
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: v_readlane_b32 s14, v21, 42
-; SI-NEXT: v_readlane_b32 s15, v21, 43
+; SI-NEXT: v_readlane_b32 s14, v22, 44
+; SI-NEXT: v_readlane_b32 s15, v22, 45
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v21, 44
+; SI-NEXT: v_readlane_b32 s16, v22, 46
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s16, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
@@ -8880,9 +9000,10 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: s_and_b32 s12, s13, 0xff
-; SI-NEXT: s_lshl_b32 s13, s39, 8
+; SI-NEXT: v_readlane_b32 s13, v22, 62
+; SI-NEXT: s_lshl_b32 s13, s13, 8
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: s_and_b32 s13, s37, 0xff
+; SI-NEXT: s_and_b32 s13, s36, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_lshl_b32 s14, s35, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
@@ -8892,15 +9013,15 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
-; SI-NEXT: v_readlane_b32 s12, v21, 46
+; SI-NEXT: v_readlane_b32 s12, v22, 48
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v21, 47
+; SI-NEXT: v_readlane_b32 s13, v22, 49
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: v_readlane_b32 s12, v21, 48
-; SI-NEXT: v_readlane_b32 s13, v21, 49
+; SI-NEXT: v_readlane_b32 s12, v22, 50
+; SI-NEXT: v_readlane_b32 s13, v22, 51
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v21, 50
+; SI-NEXT: v_readlane_b32 s14, v22, 52
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_lshl_b32 s13, s14, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
@@ -8911,9 +9032,9 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: s_and_b32 s10, s11, 0xff
-; SI-NEXT: s_lshl_b32 s11, s31, 8
+; SI-NEXT: s_lshl_b32 s11, s30, 8
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_and_b32 s11, s95, 0xff
+; SI-NEXT: s_and_b32 s11, s94, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: s_lshl_b32 s12, s93, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
@@ -8923,15 +9044,15 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_readlane_b32 s10, v21, 52
+; SI-NEXT: v_readlane_b32 s10, v22, 54
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v21, 53
+; SI-NEXT: v_readlane_b32 s11, v22, 55
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: v_readlane_b32 s10, v21, 54
-; SI-NEXT: v_readlane_b32 s11, v21, 55
+; SI-NEXT: v_readlane_b32 s10, v22, 56
+; SI-NEXT: v_readlane_b32 s11, v22, 57
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v21, 56
+; SI-NEXT: v_readlane_b32 s12, v22, 58
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s11, s12, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
@@ -8942,11 +9063,11 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: s_and_b32 s8, s9, 0xff
-; SI-NEXT: s_lshl_b32 s9, s91, 8
+; SI-NEXT: s_lshl_b32 s9, s90, 8
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_and_b32 s9, s89, 0xff
+; SI-NEXT: s_and_b32 s9, s97, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s10, s79, 24
+; SI-NEXT: s_lshl_b32 s10, s78, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x68, v0
@@ -8954,17 +9075,12 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_readlane_b32 s8, v21, 58
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v21, 59
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s98, 8
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: v_readlane_b32 s8, v21, 60
-; SI-NEXT: v_readlane_b32 s9, v21, 61
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v21, 62
+; SI-NEXT: s_and_b32 s8, s54, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s96, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x6c, v0
@@ -8973,11 +9089,11 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_and_b32 s6, s7, 0xff
-; SI-NEXT: s_lshl_b32 s7, s77, 8
+; SI-NEXT: s_lshl_b32 s7, s87, 8
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_and_b32 s7, s75, 0xff
+; SI-NEXT: s_and_b32 s7, s55, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s8, s73, 24
+; SI-NEXT: s_lshl_b32 s8, s72, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x70, v0
@@ -8985,17 +9101,12 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: v_readlane_b32 s6, v22, 0
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s7, v22, 1
-; SI-NEXT: s_lshl_b32 s6, s6, 8
+; SI-NEXT: s_lshl_b32 s6, s86, 8
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: v_readlane_b32 s6, v22, 4
-; SI-NEXT: v_readlane_b32 s7, v22, 5
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 2
+; SI-NEXT: s_and_b32 s6, s84, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s38, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x74, v0
@@ -9004,11 +9115,11 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_and_b32 s4, s5, 0xff
-; SI-NEXT: s_lshl_b32 s5, s63, 8
+; SI-NEXT: s_lshl_b32 s5, s85, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readlane_b32 s5, v22, 7
+; SI-NEXT: v_readlane_b32 s5, v22, 61
; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: v_readlane_b32 s6, v22, 6
+; SI-NEXT: v_readlane_b32 s6, v22, 60
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -9019,12 +9130,10 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v21, 39
-; SI-NEXT: v_readlane_b32 s17, v21, 45
-; SI-NEXT: v_readlane_b32 s15, v21, 51
-; SI-NEXT: v_readlane_b32 s13, v21, 57
-; SI-NEXT: v_readlane_b32 s11, v21, 63
-; SI-NEXT: v_readlane_b32 s9, v22, 3
+; SI-NEXT: v_readlane_b32 s19, v22, 41
+; SI-NEXT: v_readlane_b32 s17, v22, 47
+; SI-NEXT: v_readlane_b32 s15, v22, 53
+; SI-NEXT: v_readlane_b32 s13, v22, 59
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -9129,665 +9238,833 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_cmp_lg_u32 s44, 0
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s5, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s5, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s4, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s4, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s7, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s7, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 8
-; VI-NEXT: s_lshr_b32 s46, s7, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s6, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s6, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 11
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s9, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 13
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s8, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s11, 16
; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s11, 8
+; VI-NEXT: s_lshr_b32 s46, s4, 16
; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s10, 16
+; VI-NEXT: s_lshr_b32 s46, s7, 24
; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s10, 8
+; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s13, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s12, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s15, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s15, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s14, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s14, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s41, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s41, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s41, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s40, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 35
-; VI-NEXT: s_lshr_b32 s46, s40, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s43, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s43, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s43, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s42, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s42, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s29, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s28, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s27, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s26, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s25, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b64 s[56:57], s[4:5], 24
+; VI-NEXT: s_lshr_b32 s57, s45, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 22
+; VI-NEXT: s_lshr_b32 s57, s44, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 23
+; VI-NEXT: s_lshr_b32 s57, s44, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 24
+; VI-NEXT: s_lshr_b32 s57, s29, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 25
+; VI-NEXT: s_lshr_b32 s57, s29, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 26
+; VI-NEXT: s_lshr_b32 s56, s29, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 27
+; VI-NEXT: s_lshr_b32 s57, s28, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 28
+; VI-NEXT: s_lshr_b32 s56, s28, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 29
+; VI-NEXT: s_lshr_b32 s57, s27, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 30
+; VI-NEXT: s_lshr_b32 s57, s27, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 31
+; VI-NEXT: s_lshr_b32 s56, s27, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 32
+; VI-NEXT: s_lshr_b32 s57, s26, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 33
+; VI-NEXT: s_lshr_b32 s56, s26, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 34
+; VI-NEXT: s_lshr_b32 s57, s25, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 35
+; VI-NEXT: s_lshr_b32 s57, s25, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 36
+; VI-NEXT: s_lshr_b32 s56, s25, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 37
+; VI-NEXT: s_lshr_b32 s57, s24, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 38
+; VI-NEXT: s_lshr_b32 s56, s24, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 39
+; VI-NEXT: s_lshr_b32 s56, s23, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 40
+; VI-NEXT: s_lshr_b32 s56, s23, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 41
+; VI-NEXT: s_lshr_b32 s56, s23, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 42
+; VI-NEXT: s_lshr_b32 s56, s22, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 43
+; VI-NEXT: s_lshr_b32 s56, s22, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 44
+; VI-NEXT: s_lshr_b32 s56, s21, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 45
+; VI-NEXT: s_lshr_b32 s56, s21, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 46
+; VI-NEXT: s_lshr_b32 s56, s21, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 47
+; VI-NEXT: s_lshr_b32 s56, s20, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 48
+; VI-NEXT: s_lshr_b32 s56, s20, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 49
+; VI-NEXT: s_lshr_b32 s56, s19, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 50
+; VI-NEXT: s_lshr_b32 s56, s19, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 51
+; VI-NEXT: s_lshr_b32 s56, s19, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 52
+; VI-NEXT: s_lshr_b32 s56, s18, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 53
+; VI-NEXT: s_lshr_b32 s56, s18, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 54
+; VI-NEXT: s_lshr_b32 s56, s17, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 55
+; VI-NEXT: s_lshr_b32 s56, s17, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 56
+; VI-NEXT: s_lshr_b32 s56, s17, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 57
+; VI-NEXT: s_lshr_b32 s56, s16, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 58
+; VI-NEXT: s_lshr_b32 s56, s16, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 59
+; VI-NEXT: s_lshr_b64 s[88:89], s[4:5], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 17
+; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 14
+; VI-NEXT: v_writelane_b32 v33, s57, 15
+; VI-NEXT: s_lshr_b64 s[56:57], s[8:9], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 12
+; VI-NEXT: v_writelane_b32 v33, s57, 13
+; VI-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 10
+; VI-NEXT: v_writelane_b32 v33, s57, 11
+; VI-NEXT: s_lshr_b64 s[56:57], s[12:13], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 9
+; VI-NEXT: s_lshr_b64 s[56:57], s[14:15], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 6
+; VI-NEXT: v_writelane_b32 v33, s57, 7
+; VI-NEXT: s_lshr_b64 s[56:57], s[40:41], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 4
+; VI-NEXT: v_writelane_b32 v33, s57, 5
+; VI-NEXT: s_lshr_b64 s[56:57], s[42:43], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 2
+; VI-NEXT: v_writelane_b32 v33, s57, 3
+; VI-NEXT: s_lshr_b64 s[56:57], s[44:45], 24
; VI-NEXT: v_writelane_b32 v33, s56, 0
-; VI-NEXT: s_lshr_b32 s64, s29, 8
-; VI-NEXT: s_lshr_b32 s50, s28, 8
-; VI-NEXT: s_lshr_b32 s86, s27, 8
-; VI-NEXT: s_lshr_b32 s51, s26, 8
-; VI-NEXT: s_lshr_b32 s66, s25, 8
-; VI-NEXT: s_lshr_b32 s54, s24, 8
-; VI-NEXT: s_lshr_b32 s71, s23, 24
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s52, s23, 8
-; VI-NEXT: s_lshr_b32 s55, s22, 8
-; VI-NEXT: s_lshr_b32 s68, s21, 24
-; VI-NEXT: s_lshr_b32 s65, s21, 8
-; VI-NEXT: s_lshr_b32 s84, s20, 16
-; VI-NEXT: s_lshr_b32 s69, s20, 8
-; VI-NEXT: s_lshr_b32 s80, s19, 24
-; VI-NEXT: s_lshr_b32 s87, s19, 16
-; VI-NEXT: s_lshr_b32 s46, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s47, s18, 8
-; VI-NEXT: s_lshr_b32 s83, s17, 24
-; VI-NEXT: s_lshr_b32 s85, s17, 16
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s53, s16, 16
-; VI-NEXT: s_lshr_b32 s82, s16, 8
+; VI-NEXT: s_lshr_b32 s59, s5, 16
+; VI-NEXT: s_lshr_b32 s70, s5, 8
+; VI-NEXT: s_lshr_b32 s68, s4, 8
+; VI-NEXT: s_lshr_b32 s71, s7, 16
+; VI-NEXT: s_lshr_b32 s81, s7, 8
+; VI-NEXT: s_lshr_b32 s58, s6, 8
+; VI-NEXT: s_lshr_b32 s80, s9, 24
+; VI-NEXT: s_lshr_b32 s84, s9, 16
+; VI-NEXT: s_lshr_b32 s53, s9, 8
+; VI-NEXT: s_lshr_b32 s83, s8, 16
+; VI-NEXT: s_lshr_b32 s61, s8, 8
+; VI-NEXT: s_lshr_b32 s60, s11, 24
+; VI-NEXT: s_lshr_b32 s64, s11, 16
+; VI-NEXT: s_lshr_b32 s63, s11, 8
+; VI-NEXT: s_lshr_b32 s62, s10, 16
+; VI-NEXT: s_lshr_b32 s50, s10, 8
+; VI-NEXT: s_lshr_b32 s73, s13, 24
+; VI-NEXT: s_lshr_b32 s72, s13, 16
+; VI-NEXT: s_lshr_b32 s85, s13, 8
+; VI-NEXT: s_lshr_b32 s51, s12, 16
+; VI-NEXT: s_lshr_b32 s75, s12, 8
+; VI-NEXT: s_lshr_b32 s74, s15, 24
+; VI-NEXT: s_lshr_b32 s66, s15, 16
+; VI-NEXT: s_lshr_b32 s77, s15, 8
+; VI-NEXT: s_lshr_b32 s76, s14, 16
+; VI-NEXT: s_lshr_b32 s54, s14, 8
+; VI-NEXT: s_lshr_b32 s52, s41, 24
+; VI-NEXT: s_lshr_b32 s55, s41, 16
+; VI-NEXT: s_lshr_b32 s65, s41, 8
+; VI-NEXT: s_lshr_b32 s69, s40, 16
+; VI-NEXT: s_lshr_b32 s46, s40, 8
+; VI-NEXT: s_lshr_b32 s78, s43, 24
+; VI-NEXT: s_lshr_b32 s67, s43, 16
+; VI-NEXT: s_lshr_b32 s79, s43, 8
+; VI-NEXT: s_lshr_b32 s82, s42, 16
+; VI-NEXT: s_lshr_b32 s47, s42, 8
+; VI-NEXT: s_lshr_b32 s86, s45, 24
+; VI-NEXT: s_lshr_b32 s87, s45, 16
; VI-NEXT: v_writelane_b32 v33, s57, 1
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; kill: killed $sgpr57
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 0
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 2
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 3
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 4
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 5
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 6
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 7
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 8
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 9
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 10
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 11
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 12
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 13
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 14
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 15
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 16
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 17
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s57, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s57, 1
+; VI-NEXT: v_readlane_b32 s48, v33, 18
+; VI-NEXT: s_mov_b32 s49, s59
+; VI-NEXT: v_readlane_b32 s56, v33, 19
+; VI-NEXT: s_mov_b32 s57, s68
+; VI-NEXT: v_readlane_b32 s68, v33, 20
+; VI-NEXT: v_readlane_b32 s59, v33, 21
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
+; VI-NEXT: s_add_i32 s45, s45, 3
+; VI-NEXT: s_add_i32 s44, s44, 3
+; VI-NEXT: s_lshr_b32 s88, s45, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 22
+; VI-NEXT: s_lshr_b32 s88, s44, 16
+; VI-NEXT: s_add_i32 s29, s29, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 23
+; VI-NEXT: s_lshr_b32 s88, s44, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 24
+; VI-NEXT: s_lshr_b32 s88, s29, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 25
+; VI-NEXT: s_lshr_b32 s88, s29, 16
+; VI-NEXT: s_add_i32 s28, s28, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 26
+; VI-NEXT: s_lshr_b32 s88, s29, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 27
+; VI-NEXT: s_lshr_b32 s88, s28, 16
+; VI-NEXT: s_add_i32 s27, s27, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 28
+; VI-NEXT: s_lshr_b32 s88, s28, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 29
+; VI-NEXT: s_lshr_b32 s88, s27, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 30
+; VI-NEXT: s_lshr_b32 s88, s27, 16
+; VI-NEXT: s_add_i32 s26, s26, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 31
+; VI-NEXT: s_lshr_b32 s88, s27, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 32
+; VI-NEXT: s_lshr_b32 s88, s26, 16
+; VI-NEXT: s_add_i32 s25, s25, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 33
+; VI-NEXT: s_lshr_b32 s88, s26, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 34
+; VI-NEXT: s_lshr_b32 s88, s25, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 35
+; VI-NEXT: s_lshr_b32 s88, s25, 16
+; VI-NEXT: s_add_i32 s24, s24, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 36
+; VI-NEXT: s_lshr_b32 s88, s25, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 37
+; VI-NEXT: s_lshr_b32 s88, s24, 16
+; VI-NEXT: s_add_i32 s23, s23, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 38
+; VI-NEXT: s_lshr_b32 s88, s24, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 39
+; VI-NEXT: s_lshr_b32 s88, s23, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 40
+; VI-NEXT: s_lshr_b32 s88, s23, 16
+; VI-NEXT: s_add_i32 s22, s22, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 41
+; VI-NEXT: s_lshr_b32 s88, s23, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 42
+; VI-NEXT: s_lshr_b32 s88, s22, 16
+; VI-NEXT: s_add_i32 s21, s21, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 43
+; VI-NEXT: s_lshr_b32 s88, s22, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 44
+; VI-NEXT: s_lshr_b32 s88, s21, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 45
+; VI-NEXT: s_lshr_b32 s88, s21, 16
+; VI-NEXT: s_add_i32 s20, s20, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 46
+; VI-NEXT: s_lshr_b32 s88, s21, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 47
+; VI-NEXT: s_lshr_b32 s88, s20, 16
+; VI-NEXT: s_add_i32 s19, s19, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 48
+; VI-NEXT: s_lshr_b32 s88, s20, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 49
+; VI-NEXT: s_lshr_b32 s88, s19, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 50
+; VI-NEXT: s_lshr_b32 s88, s19, 16
+; VI-NEXT: s_add_i32 s18, s18, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 51
+; VI-NEXT: s_lshr_b32 s88, s19, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 52
+; VI-NEXT: s_lshr_b32 s88, s18, 16
+; VI-NEXT: s_add_i32 s17, s17, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 53
+; VI-NEXT: s_lshr_b32 s88, s18, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 54
+; VI-NEXT: s_lshr_b32 s88, s17, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 55
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: s_add_i32 s16, s16, 3
+; VI-NEXT: v_writelane_b32 v33, s88, 56
+; VI-NEXT: s_lshr_b32 s88, s17, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 57
+; VI-NEXT: s_lshr_b32 s88, s16, 16
; VI-NEXT: s_add_i32 s5, s5, 3
-; VI-NEXT: s_lshr_b32 s46, s5, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s5, 16
; VI-NEXT: s_add_i32 s4, s4, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s5, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s4, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 58
+; VI-NEXT: s_lshr_b32 s88, s16, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 59
+; VI-NEXT: s_lshr_b64 s[88:89], s[4:5], 24
; VI-NEXT: s_add_i32 s7, s7, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s4, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s7, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s7, 16
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 8
-; VI-NEXT: s_lshr_b32 s46, s7, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s6, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 17
+; VI-NEXT: s_lshr_b64 s[88:89], s[6:7], 24
; VI-NEXT: s_add_i32 s9, s9, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s6, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 11
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s9, 16
; VI-NEXT: s_add_i32 s8, s8, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 13
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s8, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 14
+; VI-NEXT: v_writelane_b32 v33, s89, 15
+; VI-NEXT: s_lshr_b64 s[88:89], s[8:9], 24
; VI-NEXT: s_add_i32 s11, s11, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s11, 16
; VI-NEXT: s_add_i32 s10, s10, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s11, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s10, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 12
+; VI-NEXT: v_writelane_b32 v33, s89, 13
+; VI-NEXT: s_lshr_b64 s[88:89], s[10:11], 24
; VI-NEXT: s_add_i32 s13, s13, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s10, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s13, 16
; VI-NEXT: s_add_i32 s12, s12, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s12, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 10
+; VI-NEXT: v_writelane_b32 v33, s89, 11
+; VI-NEXT: s_lshr_b64 s[88:89], s[12:13], 24
; VI-NEXT: s_add_i32 s15, s15, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s15, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s15, 16
; VI-NEXT: s_add_i32 s14, s14, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s14, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 8
+; VI-NEXT: v_writelane_b32 v33, s89, 9
+; VI-NEXT: s_lshr_b64 s[88:89], s[14:15], 24
; VI-NEXT: s_add_i32 s41, s41, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s14, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s41, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s41, 16
; VI-NEXT: s_add_i32 s40, s40, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s41, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s40, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 6
+; VI-NEXT: v_writelane_b32 v33, s89, 7
+; VI-NEXT: s_lshr_b64 s[88:89], s[40:41], 24
; VI-NEXT: s_add_i32 s43, s43, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 35
-; VI-NEXT: s_lshr_b32 s46, s40, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s43, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s43, 16
; VI-NEXT: s_add_i32 s42, s42, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s43, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s42, 16
-; VI-NEXT: s_add_i32 s45, s45, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s42, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: s_add_i32 s44, s44, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: s_add_i32 s29, s29, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: s_add_i32 s28, s28, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s29, 16
-; VI-NEXT: s_add_i32 s27, s27, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s28, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: s_add_i32 s26, s26, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s27, 16
-; VI-NEXT: s_add_i32 s25, s25, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s26, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s25, 24
-; VI-NEXT: s_add_i32 s24, s24, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: s_add_i32 s22, s22, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: s_add_i32 s21, s21, 3
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b64 s[56:57], s[4:5], 24
-; VI-NEXT: s_add_i32 s17, s17, 3
-; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: s_add_i32 s19, s19, 3
-; VI-NEXT: s_add_i32 s18, s18, 3
-; VI-NEXT: s_add_i32 s20, s20, 3
-; VI-NEXT: s_add_i32 s23, s23, 3
-; VI-NEXT: v_writelane_b32 v33, s56, 0
-; VI-NEXT: s_lshr_b32 s64, s29, 8
-; VI-NEXT: s_lshr_b32 s50, s28, 8
-; VI-NEXT: s_lshr_b32 s86, s27, 8
-; VI-NEXT: s_lshr_b32 s51, s26, 8
-; VI-NEXT: s_lshr_b32 s66, s25, 8
-; VI-NEXT: s_lshr_b32 s54, s24, 8
-; VI-NEXT: s_lshr_b32 s71, s23, 24
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s52, s23, 8
-; VI-NEXT: s_lshr_b32 s55, s22, 8
-; VI-NEXT: s_lshr_b32 s68, s21, 24
-; VI-NEXT: s_lshr_b32 s65, s21, 8
-; VI-NEXT: s_lshr_b32 s84, s20, 16
-; VI-NEXT: s_lshr_b32 s69, s20, 8
-; VI-NEXT: s_lshr_b32 s80, s19, 24
-; VI-NEXT: s_lshr_b32 s87, s19, 16
-; VI-NEXT: s_lshr_b32 s46, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s47, s18, 8
-; VI-NEXT: s_lshr_b32 s83, s17, 24
-; VI-NEXT: s_lshr_b32 s85, s17, 16
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s53, s16, 16
-; VI-NEXT: s_lshr_b32 s82, s16, 8
-; VI-NEXT: v_writelane_b32 v33, s57, 1
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 4
+; VI-NEXT: v_writelane_b32 v33, s89, 5
+; VI-NEXT: s_lshr_b64 s[88:89], s[42:43], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 2
+; VI-NEXT: v_writelane_b32 v33, s89, 3
+; VI-NEXT: s_lshr_b64 s[88:89], s[44:45], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 0
+; VI-NEXT: s_lshr_b32 s48, s5, 24
+; VI-NEXT: s_lshr_b32 s49, s5, 16
+; VI-NEXT: s_lshr_b32 s70, s5, 8
+; VI-NEXT: s_lshr_b32 s56, s4, 16
+; VI-NEXT: s_lshr_b32 s57, s4, 8
+; VI-NEXT: s_lshr_b32 s68, s7, 24
+; VI-NEXT: s_lshr_b32 s71, s7, 16
+; VI-NEXT: s_lshr_b32 s81, s7, 8
+; VI-NEXT: s_lshr_b32 s59, s6, 16
+; VI-NEXT: s_lshr_b32 s58, s6, 8
+; VI-NEXT: s_lshr_b32 s80, s9, 24
+; VI-NEXT: s_lshr_b32 s84, s9, 16
+; VI-NEXT: s_lshr_b32 s53, s9, 8
+; VI-NEXT: s_lshr_b32 s83, s8, 16
+; VI-NEXT: s_lshr_b32 s61, s8, 8
+; VI-NEXT: s_lshr_b32 s60, s11, 24
+; VI-NEXT: s_lshr_b32 s64, s11, 16
+; VI-NEXT: s_lshr_b32 s63, s11, 8
+; VI-NEXT: s_lshr_b32 s62, s10, 16
+; VI-NEXT: s_lshr_b32 s50, s10, 8
+; VI-NEXT: s_lshr_b32 s73, s13, 24
+; VI-NEXT: s_lshr_b32 s72, s13, 16
+; VI-NEXT: s_lshr_b32 s85, s13, 8
+; VI-NEXT: s_lshr_b32 s51, s12, 16
+; VI-NEXT: s_lshr_b32 s75, s12, 8
+; VI-NEXT: s_lshr_b32 s74, s15, 24
+; VI-NEXT: s_lshr_b32 s66, s15, 16
+; VI-NEXT: s_lshr_b32 s77, s15, 8
+; VI-NEXT: s_lshr_b32 s76, s14, 16
+; VI-NEXT: s_lshr_b32 s54, s14, 8
+; VI-NEXT: s_lshr_b32 s52, s41, 24
+; VI-NEXT: s_lshr_b32 s55, s41, 16
+; VI-NEXT: s_lshr_b32 s65, s41, 8
+; VI-NEXT: s_lshr_b32 s69, s40, 16
+; VI-NEXT: s_lshr_b32 s46, s40, 8
+; VI-NEXT: s_lshr_b32 s78, s43, 24
+; VI-NEXT: s_lshr_b32 s67, s43, 16
+; VI-NEXT: s_lshr_b32 s79, s43, 8
+; VI-NEXT: s_lshr_b32 s82, s42, 16
+; VI-NEXT: s_lshr_b32 s47, s42, 8
+; VI-NEXT: s_lshr_b32 s86, s45, 24
+; VI-NEXT: s_lshr_b32 s87, s45, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 1
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: .LBB13_3: ; %end
-; VI-NEXT: v_mov_b32_e32 v9, s48
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: .LBB13_5: ; %end
+; VI-NEXT: v_readlane_b32 s89, v33, 59
+; VI-NEXT: v_mov_b32_e32 v2, s89
+; VI-NEXT: v_readlane_b32 s89, v33, 57
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; VI-NEXT: v_mov_b32_e32 v2, s82
-; VI-NEXT: v_perm_b32 v9, s53, v9, v1
+; VI-NEXT: v_mov_b32_e32 v4, s89
; VI-NEXT: v_perm_b32 v3, s16, v2, v1
-; VI-NEXT: v_mov_b32_e32 v24, s83
+; VI-NEXT: v_readlane_b32 s16, v33, 54
+; VI-NEXT: v_perm_b32 v2, s17, v4, v1
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 52
+; VI-NEXT: v_mov_b32_e32 v6, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 49
+; VI-NEXT: v_perm_b32 v5, s18, v4, v1
+; VI-NEXT: v_perm_b32 v4, s19, v6, v1
+; VI-NEXT: v_mov_b32_e32 v6, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 47
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 44
+; VI-NEXT: v_perm_b32 v7, s20, v6, v1
+; VI-NEXT: v_perm_b32 v6, s21, v8, v1
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 42
+; VI-NEXT: v_mov_b32_e32 v11, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 39
+; VI-NEXT: v_perm_b32 v10, s22, v8, v1
+; VI-NEXT: v_perm_b32 v8, s23, v11, v1
+; VI-NEXT: v_mov_b32_e32 v11, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 37
+; VI-NEXT: v_mov_b32_e32 v14, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 34
+; VI-NEXT: v_perm_b32 v12, s24, v11, v1
+; VI-NEXT: v_perm_b32 v11, s25, v14, v1
+; VI-NEXT: v_mov_b32_e32 v14, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 32
+; VI-NEXT: v_mov_b32_e32 v17, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 29
+; VI-NEXT: v_perm_b32 v15, s26, v14, v1
+; VI-NEXT: v_perm_b32 v14, s27, v17, v1
+; VI-NEXT: v_mov_b32_e32 v17, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 27
+; VI-NEXT: v_mov_b32_e32 v23, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 0
+; VI-NEXT: v_mov_b32_e32 v9, s38
+; VI-NEXT: v_perm_b32 v18, s28, v17, v1
+; VI-NEXT: v_perm_b32 v17, s29, v23, v1
+; VI-NEXT: v_mov_b32_e32 v23, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 58
+; VI-NEXT: v_perm_b32 v9, s16, v9, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 55
+; VI-NEXT: v_mov_b32_e32 v24, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 56
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; VI-NEXT: v_mov_b32_e32 v4, s67
-; VI-NEXT: v_perm_b32 v24, s85, v24, v1
+; VI-NEXT: v_perm_b32 v24, s16, v24, v1
; VI-NEXT: v_or_b32_e32 v3, v3, v9
-; VI-NEXT: v_perm_b32 v2, s17, v4, v1
-; VI-NEXT: v_mov_b32_e32 v13, s38
+; VI-NEXT: v_mov_b32_e32 v13, s36
+; VI-NEXT: v_readlane_b32 s16, v33, 53
; VI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; VI-NEXT: v_mov_b32_e32 v4, s47
-; VI-NEXT: v_perm_b32 v13, s81, v13, v1
+; VI-NEXT: v_mov_b32_e32 v20, vcc_lo
+; VI-NEXT: v_perm_b32 v13, s16, v13, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 50
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 4, v0
-; VI-NEXT: v_perm_b32 v5, s18, v4, v1
-; VI-NEXT: v_mov_b32_e32 v25, s80
+; VI-NEXT: v_mov_b32_e32 v25, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 51
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
-; VI-NEXT: v_mov_b32_e32 v6, s46
-; VI-NEXT: v_perm_b32 v25, s87, v25, v1
+; VI-NEXT: v_perm_b32 v25, s16, v25, v1
; VI-NEXT: v_or_b32_e32 v2, v5, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 8, v0
-; VI-NEXT: v_perm_b32 v4, s19, v6, v1
-; VI-NEXT: v_mov_b32_e32 v17, s36
+; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_readlane_b32 s16, v33, 48
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; VI-NEXT: v_mov_b32_e32 v6, s69
-; VI-NEXT: v_perm_b32 v17, s84, v17, v1
+; VI-NEXT: v_perm_b32 v16, s16, v16, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 45
; VI-NEXT: v_or_b32_e32 v2, v4, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 12, v0
-; VI-NEXT: v_perm_b32 v7, s20, v6, v1
-; VI-NEXT: v_mov_b32_e32 v26, s68
-; VI-NEXT: v_readlane_b32 s16, v33, 57
+; VI-NEXT: v_mov_b32_e32 v26, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 46
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v17
-; VI-NEXT: v_mov_b32_e32 v8, s65
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
; VI-NEXT: v_perm_b32 v26, s16, v26, v1
; VI-NEXT: v_or_b32_e32 v2, v7, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT: v_perm_b32 v6, s21, v8, v1
-; VI-NEXT: v_mov_b32_e32 v19, s34
-; VI-NEXT: v_readlane_b32 s16, v33, 56
+; VI-NEXT: v_mov_b32_e32 v19, s30
+; VI-NEXT: v_readlane_b32 s16, v33, 43
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v26
-; VI-NEXT: v_mov_b32_e32 v8, s55
; VI-NEXT: v_perm_b32 v19, s16, v19, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 40
; VI-NEXT: v_or_b32_e32 v2, v6, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 20, v0
-; VI-NEXT: v_perm_b32 v10, s22, v8, v1
-; VI-NEXT: v_mov_b32_e32 v27, s71
+; VI-NEXT: v_mov_b32_e32 v27, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 41
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v19
-; VI-NEXT: v_mov_b32_e32 v11, s52
-; VI-NEXT: v_perm_b32 v27, s70, v27, v1
+; VI-NEXT: v_perm_b32 v27, s16, v27, v1
; VI-NEXT: v_or_b32_e32 v2, v10, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 24, v0
-; VI-NEXT: v_perm_b32 v8, s23, v11, v1
-; VI-NEXT: v_mov_b32_e32 v20, s30
-; VI-NEXT: v_readlane_b32 s16, v33, 55
+; VI-NEXT: v_readlane_b32 s16, v33, 38
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v27
-; VI-NEXT: v_mov_b32_e32 v11, s54
; VI-NEXT: v_perm_b32 v20, s16, v20, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 53
+; VI-NEXT: v_readlane_b32 s16, v33, 35
; VI-NEXT: v_or_b32_e32 v2, v8, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 28, v0
-; VI-NEXT: v_perm_b32 v12, s24, v11, v1
; VI-NEXT: v_mov_b32_e32 v28, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 54
+; VI-NEXT: v_readlane_b32 s16, v33, 36
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v20
-; VI-NEXT: v_mov_b32_e32 v14, s66
; VI-NEXT: v_perm_b32 v28, s16, v28, v1
; VI-NEXT: v_or_b32_e32 v2, v12, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 32, v0
-; VI-NEXT: v_perm_b32 v11, s25, v14, v1
; VI-NEXT: v_mov_b32_e32 v21, s90
-; VI-NEXT: v_readlane_b32 s16, v33, 52
+; VI-NEXT: v_readlane_b32 s16, v33, 33
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v28
-; VI-NEXT: v_mov_b32_e32 v14, s51
; VI-NEXT: v_perm_b32 v21, s16, v21, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 50
+; VI-NEXT: v_readlane_b32 s16, v33, 30
; VI-NEXT: v_or_b32_e32 v2, v11, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 36, v0
-; VI-NEXT: v_perm_b32 v15, s26, v14, v1
; VI-NEXT: v_mov_b32_e32 v29, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 51
+; VI-NEXT: v_readlane_b32 s16, v33, 31
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v21
-; VI-NEXT: v_mov_b32_e32 v16, s86
; VI-NEXT: v_perm_b32 v29, s16, v29, v1
; VI-NEXT: v_or_b32_e32 v2, v15, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 40, v0
-; VI-NEXT: v_perm_b32 v14, s27, v16, v1
; VI-NEXT: v_mov_b32_e32 v22, s88
-; VI-NEXT: v_readlane_b32 s16, v33, 49
+; VI-NEXT: v_readlane_b32 s16, v33, 28
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
-; VI-NEXT: v_mov_b32_e32 v16, s50
; VI-NEXT: v_perm_b32 v22, s16, v22, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 47
+; VI-NEXT: v_readlane_b32 s16, v33, 25
; VI-NEXT: v_or_b32_e32 v2, v14, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 44, v0
-; VI-NEXT: v_perm_b32 v18, s28, v16, v1
; VI-NEXT: v_mov_b32_e32 v30, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 48
+; VI-NEXT: v_readlane_b32 s16, v33, 26
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; VI-NEXT: v_mov_b32_e32 v23, s64
; VI-NEXT: v_perm_b32 v30, s16, v30, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 46
+; VI-NEXT: v_readlane_b32 s16, v33, 24
; VI-NEXT: v_or_b32_e32 v2, v18, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 48, v0
-; VI-NEXT: v_perm_b32 v16, s29, v23, v1
-; VI-NEXT: v_mov_b32_e32 v23, s78
; VI-NEXT: v_mov_b32_e32 v31, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 45
+; VI-NEXT: v_readlane_b32 s16, v33, 23
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
; VI-NEXT: v_perm_b32 v23, s16, v23, v1
-; VI-NEXT: v_or_b32_e32 v2, v16, v2
+; VI-NEXT: v_or_b32_e32 v2, v17, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 52, v0
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_perm_b32 v2, s44, v31, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v23
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 56, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 44
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
+; VI-NEXT: v_readlane_b32 s16, v33, 22
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 42
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 43
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_perm_b32 v3, s87, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 1
; VI-NEXT: v_perm_b32 v2, s45, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 41
+; VI-NEXT: v_readlane_b32 s16, v33, 2
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_mov_b32_e32 v3, s76
-; VI-NEXT: v_readlane_b32 s16, v33, 40
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s47
+; VI-NEXT: v_perm_b32 v3, s82, v3, v1
; VI-NEXT: v_perm_b32 v2, s42, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 64, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 39
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 37
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 38
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s78
+; VI-NEXT: v_mov_b32_e32 v2, s79
+; VI-NEXT: v_perm_b32 v3, s67, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 3
; VI-NEXT: v_perm_b32 v2, s43, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x44, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 36
+; VI-NEXT: v_readlane_b32 s16, v33, 4
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_mov_b32_e32 v3, s74
-; VI-NEXT: v_readlane_b32 s16, v33, 35
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s46
+; VI-NEXT: v_perm_b32 v3, s69, v3, v1
; VI-NEXT: v_perm_b32 v2, s40, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x48, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 34
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 32
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 33
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s52
+; VI-NEXT: v_mov_b32_e32 v2, s65
+; VI-NEXT: v_perm_b32 v3, s55, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 5
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x4c, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 31
+; VI-NEXT: v_readlane_b32 s16, v33, 6
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s54
+; VI-NEXT: v_perm_b32 v3, s76, v3, v1
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s72
-; VI-NEXT: v_readlane_b32 s14, v33, 30
-; VI-NEXT: v_perm_b32 v3, s14, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x50, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 29
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 27
-; VI-NEXT: v_mov_b32_e32 v3, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 28
-; VI-NEXT: v_perm_b32 v3, s14, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s74
+; VI-NEXT: v_mov_b32_e32 v2, s77
+; VI-NEXT: v_perm_b32 v3, s66, v3, v1
; VI-NEXT: v_perm_b32 v2, s15, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x54, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 26
+; VI-NEXT: v_readlane_b32 s14, v33, 8
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s14
+; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_perm_b32 v3, s51, v3, v1
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s62
-; VI-NEXT: v_readlane_b32 s12, v33, 25
-; VI-NEXT: v_perm_b32 v3, s12, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x58, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 24
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 22
-; VI-NEXT: v_mov_b32_e32 v3, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 23
-; VI-NEXT: v_perm_b32 v3, s12, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s73
+; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_perm_b32 v3, s72, v3, v1
; VI-NEXT: v_perm_b32 v2, s13, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x5c, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 21
+; VI-NEXT: v_readlane_b32 s12, v33, 10
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s12
+; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_mov_b32_e32 v2, s50
+; VI-NEXT: v_perm_b32 v3, s62, v3, v1
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s60
-; VI-NEXT: v_readlane_b32 s10, v33, 20
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x60, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 19
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 17
-; VI-NEXT: v_mov_b32_e32 v3, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 18
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s60
+; VI-NEXT: v_mov_b32_e32 v2, s63
+; VI-NEXT: v_perm_b32 v3, s64, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x64, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 16
+; VI-NEXT: v_readlane_b32 s10, v33, 12
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: v_mov_b32_e32 v3, s10
+; VI-NEXT: v_mov_b32_e32 v2, s61
+; VI-NEXT: v_perm_b32 v3, s83, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s58
-; VI-NEXT: v_readlane_b32 s8, v33, 15
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x68, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 12
-; VI-NEXT: v_mov_b32_e32 v3, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 13
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v2, s53
+; VI-NEXT: v_perm_b32 v3, s84, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x6c, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 11
+; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: v_mov_b32_e32 v2, s58
+; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s56
-; VI-NEXT: v_readlane_b32 s6, v33, 10
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x70, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 9
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 7
-; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 8
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_perm_b32 v3, s71, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x74, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 6
+; VI-NEXT: v_readlane_b32 s6, v33, 16
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 0
-; VI-NEXT: v_perm_b32 v2, s4, v2, v1
; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_readlane_b32 s4, v33, 5
-; VI-NEXT: v_perm_b32 v3, s4, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s57
+; VI-NEXT: v_perm_b32 v3, s56, v3, v1
+; VI-NEXT: v_perm_b32 v2, s4, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x78, v0
-; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 2
-; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v3, s48
; VI-NEXT: v_perm_b32 v2, s5, v2, v1
-; VI-NEXT: v_perm_b32 v1, s4, v3, v1
+; VI-NEXT: v_perm_b32 v1, s49, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s7, v33, 1
+; VI-NEXT: v_readlane_b32 s17, v33, 7
+; VI-NEXT: v_readlane_b32 s15, v33, 9
+; VI-NEXT: v_readlane_b32 s13, v33, 11
+; VI-NEXT: v_readlane_b32 s11, v33, 13
+; VI-NEXT: v_readlane_b32 s9, v33, 15
+; VI-NEXT: v_readlane_b32 s7, v33, 17
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -9826,162 +10103,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: v_writelane_b32 v33, s70, 0
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: v_writelane_b32 v33, s71, 1
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v32i32_to_v128i8_scalar:
; GFX9: ; %bb.0:
@@ -10047,610 +10168,769 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_cmp_lg_u32 s44, 0
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: ; implicit-def: $vgpr30 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 0
-; GFX9-NEXT: s_lshr_b32 s46, s5, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 1
-; GFX9-NEXT: s_lshr_b32 s46, s5, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 2
-; GFX9-NEXT: s_lshr_b32 s46, s4, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 3
-; GFX9-NEXT: s_lshr_b32 s46, s4, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 4
-; GFX9-NEXT: s_lshr_b32 s46, s7, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 5
-; GFX9-NEXT: s_lshr_b32 s46, s7, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 6
-; GFX9-NEXT: s_lshr_b32 s46, s7, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 7
-; GFX9-NEXT: s_lshr_b32 s46, s6, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 8
-; GFX9-NEXT: s_lshr_b32 s46, s6, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 9
-; GFX9-NEXT: s_lshr_b32 s46, s9, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 10
-; GFX9-NEXT: s_lshr_b32 s46, s9, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 11
-; GFX9-NEXT: s_lshr_b32 s46, s9, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 12
-; GFX9-NEXT: s_lshr_b32 s46, s8, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 13
-; GFX9-NEXT: s_lshr_b32 s46, s8, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 14
-; GFX9-NEXT: s_lshr_b32 s46, s11, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 15
-; GFX9-NEXT: s_lshr_b32 s46, s11, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 16
-; GFX9-NEXT: s_lshr_b32 s46, s11, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 17
-; GFX9-NEXT: s_lshr_b32 s46, s10, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 18
-; GFX9-NEXT: s_lshr_b32 s46, s10, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 19
-; GFX9-NEXT: s_lshr_b32 s46, s13, 24
; GFX9-NEXT: v_writelane_b32 v30, s46, 20
-; GFX9-NEXT: s_lshr_b32 s46, s13, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 21
-; GFX9-NEXT: s_lshr_b32 s46, s13, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 22
-; GFX9-NEXT: s_lshr_b32 s46, s12, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 23
-; GFX9-NEXT: s_lshr_b32 s46, s12, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 24
-; GFX9-NEXT: s_lshr_b32 s46, s15, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 25
-; GFX9-NEXT: s_lshr_b32 s46, s15, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 26
-; GFX9-NEXT: s_lshr_b32 s46, s15, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 27
-; GFX9-NEXT: s_lshr_b32 s46, s14, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 28
-; GFX9-NEXT: s_lshr_b32 s46, s14, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 29
-; GFX9-NEXT: s_lshr_b32 s46, s41, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 30
-; GFX9-NEXT: s_lshr_b32 s46, s41, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 31
-; GFX9-NEXT: s_lshr_b32 s46, s41, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 32
-; GFX9-NEXT: s_lshr_b32 s46, s40, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 33
-; GFX9-NEXT: s_lshr_b32 s46, s40, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 34
-; GFX9-NEXT: s_lshr_b32 s46, s43, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 35
-; GFX9-NEXT: s_lshr_b32 s46, s43, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 36
-; GFX9-NEXT: s_lshr_b32 s46, s43, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 37
-; GFX9-NEXT: s_lshr_b32 s46, s42, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 38
-; GFX9-NEXT: s_lshr_b32 s46, s42, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 39
-; GFX9-NEXT: s_lshr_b32 s46, s45, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 40
-; GFX9-NEXT: s_lshr_b32 s46, s45, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 41
-; GFX9-NEXT: s_lshr_b32 s46, s45, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 42
-; GFX9-NEXT: s_lshr_b32 s46, s44, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 43
-; GFX9-NEXT: s_lshr_b32 s46, s44, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 44
-; GFX9-NEXT: s_lshr_b32 s46, s29, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 45
-; GFX9-NEXT: s_lshr_b32 s46, s29, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 46
-; GFX9-NEXT: s_lshr_b32 s46, s29, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 47
-; GFX9-NEXT: s_lshr_b32 s46, s28, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 48
-; GFX9-NEXT: s_lshr_b32 s46, s27, 16
-; GFX9-NEXT: s_lshr_b32 s83, s28, 8
-; GFX9-NEXT: s_lshr_b32 s81, s27, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 49
-; GFX9-NEXT: s_lshr_b32 s53, s27, 8
-; GFX9-NEXT: s_lshr_b32 s82, s26, 16
-; GFX9-NEXT: s_lshr_b32 s67, s26, 8
-; GFX9-NEXT: s_lshr_b32 s84, s25, 24
-; GFX9-NEXT: s_lshr_b32 s96, s25, 16
-; GFX9-NEXT: s_lshr_b32 s65, s25, 8
-; GFX9-NEXT: s_lshr_b32 s86, s24, 16
-; GFX9-NEXT: s_lshr_b32 s55, s24, 8
-; GFX9-NEXT: s_lshr_b32 s98, s23, 24
-; GFX9-NEXT: s_lshr_b32 s87, s23, 16
-; GFX9-NEXT: s_lshr_b32 s66, s23, 8
-; GFX9-NEXT: s_lshr_b32 s97, s22, 16
-; GFX9-NEXT: s_lshr_b32 s64, s22, 8
-; GFX9-NEXT: s_lshr_b32 s49, s21, 24
-; GFX9-NEXT: s_lshr_b32 s99, s21, 16
-; GFX9-NEXT: s_lshr_b32 s80, s21, 8
-; GFX9-NEXT: s_lshr_b32 s39, s20, 16
-; GFX9-NEXT: s_lshr_b32 s70, s20, 8
-; GFX9-NEXT: s_lshr_b32 s48, s19, 24
-; GFX9-NEXT: s_lshr_b32 s38, s19, 16
-; GFX9-NEXT: s_lshr_b32 s68, s19, 8
-; GFX9-NEXT: s_lshr_b32 s54, s18, 16
-; GFX9-NEXT: s_lshr_b32 s71, s18, 8
-; GFX9-NEXT: s_lshr_b32 s52, s17, 24
-; GFX9-NEXT: s_lshr_b32 s50, s17, 16
-; GFX9-NEXT: s_lshr_b32 s69, s17, 8
-; GFX9-NEXT: s_lshr_b32 s51, s16, 16
-; GFX9-NEXT: s_lshr_b32 s85, s16, 8
-; GFX9-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; GFX9-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; GFX9-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; GFX9-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; GFX9-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; GFX9-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; GFX9-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
+; GFX9-NEXT: s_lshr_b32 s47, s11, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 21
+; GFX9-NEXT: s_lshr_b32 s47, s10, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 22
+; GFX9-NEXT: s_lshr_b32 s47, s27, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 23
+; GFX9-NEXT: s_lshr_b32 s47, s26, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 24
+; GFX9-NEXT: s_lshr_b32 s47, s26, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 25
+; GFX9-NEXT: s_lshr_b32 s47, s25, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 26
+; GFX9-NEXT: s_lshr_b32 s47, s25, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 27
+; GFX9-NEXT: s_lshr_b32 s47, s25, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 28
+; GFX9-NEXT: s_lshr_b32 s47, s24, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 29
+; GFX9-NEXT: s_lshr_b32 s47, s24, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 30
+; GFX9-NEXT: s_lshr_b32 s47, s23, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 31
+; GFX9-NEXT: s_lshr_b32 s47, s23, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 32
+; GFX9-NEXT: s_lshr_b32 s47, s23, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 33
+; GFX9-NEXT: s_lshr_b32 s47, s22, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 34
+; GFX9-NEXT: s_lshr_b32 s47, s22, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 35
+; GFX9-NEXT: s_lshr_b32 s47, s21, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 36
+; GFX9-NEXT: s_lshr_b32 s47, s21, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 37
+; GFX9-NEXT: s_lshr_b32 s47, s21, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 38
+; GFX9-NEXT: s_lshr_b32 s47, s20, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 39
+; GFX9-NEXT: s_lshr_b32 s47, s20, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 40
+; GFX9-NEXT: s_lshr_b32 s47, s19, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 41
+; GFX9-NEXT: s_lshr_b32 s47, s19, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 42
+; GFX9-NEXT: s_lshr_b32 s47, s19, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 43
+; GFX9-NEXT: s_lshr_b32 s47, s18, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 44
+; GFX9-NEXT: s_lshr_b32 s47, s18, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 45
+; GFX9-NEXT: s_lshr_b32 s47, s17, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 46
+; GFX9-NEXT: s_lshr_b32 s47, s17, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 47
+; GFX9-NEXT: s_lshr_b32 s47, s17, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 48
+; GFX9-NEXT: s_lshr_b32 s47, s16, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 49
+; GFX9-NEXT: s_lshr_b32 s47, s16, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 50
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[4:5], 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[8:9], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 16
+; GFX9-NEXT: v_writelane_b32 v30, s93, 17
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[10:11], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 14
+; GFX9-NEXT: v_writelane_b32 v30, s93, 15
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 12
+; GFX9-NEXT: v_writelane_b32 v30, s93, 13
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 10
+; GFX9-NEXT: v_writelane_b32 v30, s93, 11
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[40:41], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 8
+; GFX9-NEXT: v_writelane_b32 v30, s93, 9
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[42:43], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 6
+; GFX9-NEXT: v_writelane_b32 v30, s93, 7
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[44:45], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 4
+; GFX9-NEXT: v_writelane_b32 v30, s93, 5
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[28:29], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 2
+; GFX9-NEXT: v_writelane_b32 v30, s93, 3
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[26:27], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 0
+; GFX9-NEXT: s_lshr_b32 s81, s5, 16
+; GFX9-NEXT: s_lshr_b32 s82, s5, 8
+; GFX9-NEXT: s_lshr_b32 s96, s4, 16
+; GFX9-NEXT: s_lshr_b32 s83, s4, 8
+; GFX9-NEXT: s_lshr_b32 s86, s7, 24
+; GFX9-NEXT: s_lshr_b32 s84, s7, 16
+; GFX9-NEXT: s_lshr_b32 s87, s7, 8
+; GFX9-NEXT: s_lshr_b32 s57, s6, 16
+; GFX9-NEXT: s_lshr_b32 s97, s6, 8
+; GFX9-NEXT: s_lshr_b32 s98, s9, 24
+; GFX9-NEXT: s_lshr_b32 s46, s9, 16
+; GFX9-NEXT: s_lshr_b32 s48, s9, 8
+; GFX9-NEXT: s_lshr_b32 s39, s8, 16
+; GFX9-NEXT: s_lshr_b32 s38, s8, 8
+; GFX9-NEXT: s_lshr_b32 s56, s11, 16
+; GFX9-NEXT: s_lshr_b32 s49, s11, 8
+; GFX9-NEXT: s_lshr_b32 s50, s10, 8
+; GFX9-NEXT: s_lshr_b32 s51, s13, 24
+; GFX9-NEXT: s_lshr_b32 s52, s13, 16
+; GFX9-NEXT: s_lshr_b32 s59, s13, 8
+; GFX9-NEXT: s_lshr_b32 s58, s12, 16
+; GFX9-NEXT: s_lshr_b32 s67, s12, 8
+; GFX9-NEXT: s_lshr_b32 s60, s15, 24
+; GFX9-NEXT: s_lshr_b32 s65, s15, 16
+; GFX9-NEXT: s_lshr_b32 s61, s15, 8
+; GFX9-NEXT: s_lshr_b32 s62, s14, 16
+; GFX9-NEXT: s_lshr_b32 s55, s14, 8
+; GFX9-NEXT: s_lshr_b32 s63, s41, 24
+; GFX9-NEXT: s_lshr_b32 s72, s41, 16
+; GFX9-NEXT: s_lshr_b32 s66, s41, 8
+; GFX9-NEXT: s_lshr_b32 s73, s40, 16
+; GFX9-NEXT: s_lshr_b32 s64, s40, 8
+; GFX9-NEXT: s_lshr_b32 s74, s43, 24
+; GFX9-NEXT: s_lshr_b32 s80, s43, 16
+; GFX9-NEXT: s_lshr_b32 s75, s43, 8
+; GFX9-NEXT: s_lshr_b32 s76, s42, 16
+; GFX9-NEXT: s_lshr_b32 s70, s42, 8
+; GFX9-NEXT: s_lshr_b32 s77, s45, 24
+; GFX9-NEXT: s_lshr_b32 s78, s45, 16
+; GFX9-NEXT: s_lshr_b32 s68, s45, 8
+; GFX9-NEXT: s_lshr_b32 s79, s44, 16
+; GFX9-NEXT: s_lshr_b32 s71, s44, 8
+; GFX9-NEXT: s_lshr_b32 s69, s29, 24
+; GFX9-NEXT: s_lshr_b32 s85, s29, 16
+; GFX9-NEXT: s_lshr_b32 s88, s29, 8
+; GFX9-NEXT: s_lshr_b32 s54, s28, 16
+; GFX9-NEXT: s_lshr_b32 s89, s28, 8
+; GFX9-NEXT: s_lshr_b32 s99, s27, 24
+; GFX9-NEXT: s_lshr_b32 s53, s27, 16
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[6:7], 24
+; GFX9-NEXT: v_writelane_b32 v30, s93, 1
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 0
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; kill: killed $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 2
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 3
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 4
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 5
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 6
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 7
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 8
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 9
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 10
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 11
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 12
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 13
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 14
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 15
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 16
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 17
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: v_readlane_b32 s36, v30, 20
+; GFX9-NEXT: s_mov_b32 s37, s81
+; GFX9-NEXT: s_mov_b32 s81, s96
+; GFX9-NEXT: s_mov_b32 s96, s57
+; GFX9-NEXT: s_mov_b32 s47, s48
+; GFX9-NEXT: v_readlane_b32 s57, v30, 21
+; GFX9-NEXT: v_readlane_b32 s48, v30, 22
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_add_i32 s27, s27, 3
+; GFX9-NEXT: s_add_i32 s26, s26, 3
+; GFX9-NEXT: s_lshr_b32 s90, s27, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 23
+; GFX9-NEXT: s_lshr_b32 s90, s26, 16
+; GFX9-NEXT: s_add_i32 s25, s25, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 24
+; GFX9-NEXT: s_lshr_b32 s90, s26, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 25
+; GFX9-NEXT: s_lshr_b32 s90, s25, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 26
+; GFX9-NEXT: s_lshr_b32 s90, s25, 16
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 27
+; GFX9-NEXT: s_lshr_b32 s90, s25, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 28
+; GFX9-NEXT: s_lshr_b32 s90, s24, 16
+; GFX9-NEXT: s_add_i32 s23, s23, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 29
+; GFX9-NEXT: s_lshr_b32 s90, s24, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 30
+; GFX9-NEXT: s_lshr_b32 s90, s23, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 31
+; GFX9-NEXT: s_lshr_b32 s90, s23, 16
+; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 32
+; GFX9-NEXT: s_lshr_b32 s90, s23, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 33
+; GFX9-NEXT: s_lshr_b32 s90, s22, 16
+; GFX9-NEXT: s_add_i32 s21, s21, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 34
+; GFX9-NEXT: s_lshr_b32 s90, s22, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 35
+; GFX9-NEXT: s_lshr_b32 s90, s21, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 36
+; GFX9-NEXT: s_lshr_b32 s90, s21, 16
+; GFX9-NEXT: s_add_i32 s20, s20, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 37
+; GFX9-NEXT: s_lshr_b32 s90, s21, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 38
+; GFX9-NEXT: s_lshr_b32 s90, s20, 16
+; GFX9-NEXT: s_add_i32 s19, s19, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 39
+; GFX9-NEXT: s_lshr_b32 s90, s20, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 40
+; GFX9-NEXT: s_lshr_b32 s90, s19, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 41
+; GFX9-NEXT: s_lshr_b32 s90, s19, 16
+; GFX9-NEXT: s_add_i32 s18, s18, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 42
+; GFX9-NEXT: s_lshr_b32 s90, s19, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 43
+; GFX9-NEXT: s_lshr_b32 s90, s18, 16
+; GFX9-NEXT: s_add_i32 s17, s17, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 44
+; GFX9-NEXT: s_lshr_b32 s90, s18, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 45
+; GFX9-NEXT: s_lshr_b32 s90, s17, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 46
+; GFX9-NEXT: s_lshr_b32 s90, s17, 16
+; GFX9-NEXT: s_add_i32 s16, s16, 3
+; GFX9-NEXT: v_writelane_b32 v30, s90, 47
+; GFX9-NEXT: s_lshr_b32 s90, s17, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 48
+; GFX9-NEXT: s_lshr_b32 s90, s16, 16
; GFX9-NEXT: s_add_i32 s5, s5, 3
-; GFX9-NEXT: s_lshr_b32 s46, s5, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 0
-; GFX9-NEXT: s_lshr_b32 s46, s5, 16
; GFX9-NEXT: s_add_i32 s4, s4, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 1
-; GFX9-NEXT: s_lshr_b32 s46, s5, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 2
-; GFX9-NEXT: s_lshr_b32 s46, s4, 16
-; GFX9-NEXT: s_add_i32 s7, s7, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 3
-; GFX9-NEXT: s_lshr_b32 s46, s4, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 4
-; GFX9-NEXT: s_lshr_b32 s46, s7, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 5
-; GFX9-NEXT: s_lshr_b32 s46, s7, 16
-; GFX9-NEXT: s_add_i32 s6, s6, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 6
-; GFX9-NEXT: s_lshr_b32 s46, s7, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 7
-; GFX9-NEXT: s_lshr_b32 s46, s6, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 49
+; GFX9-NEXT: s_lshr_b32 s90, s16, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 50
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[4:5], 24
; GFX9-NEXT: s_add_i32 s9, s9, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 8
-; GFX9-NEXT: s_lshr_b32 s46, s6, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 9
-; GFX9-NEXT: s_lshr_b32 s46, s9, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 10
-; GFX9-NEXT: s_lshr_b32 s46, s9, 16
; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 11
-; GFX9-NEXT: s_lshr_b32 s46, s9, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 12
-; GFX9-NEXT: s_lshr_b32 s46, s8, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[8:9], 24
; GFX9-NEXT: s_add_i32 s11, s11, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 13
-; GFX9-NEXT: s_lshr_b32 s46, s8, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 14
-; GFX9-NEXT: s_lshr_b32 s46, s11, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 15
-; GFX9-NEXT: s_lshr_b32 s46, s11, 16
; GFX9-NEXT: s_add_i32 s10, s10, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 16
-; GFX9-NEXT: s_lshr_b32 s46, s11, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 17
-; GFX9-NEXT: s_lshr_b32 s46, s10, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 16
+; GFX9-NEXT: v_writelane_b32 v30, s93, 17
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[10:11], 24
; GFX9-NEXT: s_add_i32 s13, s13, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 18
-; GFX9-NEXT: s_lshr_b32 s46, s10, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 19
-; GFX9-NEXT: s_lshr_b32 s46, s13, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 20
-; GFX9-NEXT: s_lshr_b32 s46, s13, 16
; GFX9-NEXT: s_add_i32 s12, s12, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 21
-; GFX9-NEXT: s_lshr_b32 s46, s13, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 22
-; GFX9-NEXT: s_lshr_b32 s46, s12, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 14
+; GFX9-NEXT: v_writelane_b32 v30, s93, 15
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
; GFX9-NEXT: s_add_i32 s15, s15, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 23
-; GFX9-NEXT: s_lshr_b32 s46, s12, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 24
-; GFX9-NEXT: s_lshr_b32 s46, s15, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 25
-; GFX9-NEXT: s_lshr_b32 s46, s15, 16
; GFX9-NEXT: s_add_i32 s14, s14, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 26
-; GFX9-NEXT: s_lshr_b32 s46, s15, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 27
-; GFX9-NEXT: s_lshr_b32 s46, s14, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 12
+; GFX9-NEXT: v_writelane_b32 v30, s93, 13
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
; GFX9-NEXT: s_add_i32 s41, s41, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 28
-; GFX9-NEXT: s_lshr_b32 s46, s14, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 29
-; GFX9-NEXT: s_lshr_b32 s46, s41, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 30
-; GFX9-NEXT: s_lshr_b32 s46, s41, 16
; GFX9-NEXT: s_add_i32 s40, s40, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 31
-; GFX9-NEXT: s_lshr_b32 s46, s41, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 32
-; GFX9-NEXT: s_lshr_b32 s46, s40, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 10
+; GFX9-NEXT: v_writelane_b32 v30, s93, 11
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[40:41], 24
; GFX9-NEXT: s_add_i32 s43, s43, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 33
-; GFX9-NEXT: s_lshr_b32 s46, s40, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 34
-; GFX9-NEXT: s_lshr_b32 s46, s43, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 35
-; GFX9-NEXT: s_lshr_b32 s46, s43, 16
; GFX9-NEXT: s_add_i32 s42, s42, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 36
-; GFX9-NEXT: s_lshr_b32 s46, s43, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 37
-; GFX9-NEXT: s_lshr_b32 s46, s42, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 8
+; GFX9-NEXT: v_writelane_b32 v30, s93, 9
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[42:43], 24
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 38
-; GFX9-NEXT: s_lshr_b32 s46, s42, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 39
-; GFX9-NEXT: s_lshr_b32 s46, s45, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 40
-; GFX9-NEXT: s_lshr_b32 s46, s45, 16
; GFX9-NEXT: s_add_i32 s44, s44, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 41
-; GFX9-NEXT: s_lshr_b32 s46, s45, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 42
-; GFX9-NEXT: s_lshr_b32 s46, s44, 16
+; GFX9-NEXT: v_writelane_b32 v30, s92, 6
+; GFX9-NEXT: v_writelane_b32 v30, s93, 7
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[44:45], 24
; GFX9-NEXT: s_add_i32 s29, s29, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 43
-; GFX9-NEXT: s_lshr_b32 s46, s44, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 44
-; GFX9-NEXT: s_lshr_b32 s46, s29, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 45
-; GFX9-NEXT: s_lshr_b32 s46, s29, 16
; GFX9-NEXT: s_add_i32 s28, s28, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 46
-; GFX9-NEXT: s_lshr_b32 s46, s29, 8
-; GFX9-NEXT: s_add_i32 s27, s27, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 47
-; GFX9-NEXT: s_lshr_b32 s46, s28, 16
-; GFX9-NEXT: s_add_i32 s17, s17, 3
-; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: s_add_i32 s19, s19, 3
-; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: s_add_i32 s21, s21, 3
-; GFX9-NEXT: s_add_i32 s20, s20, 3
-; GFX9-NEXT: s_add_i32 s23, s23, 3
-; GFX9-NEXT: s_add_i32 s22, s22, 3
-; GFX9-NEXT: s_add_i32 s25, s25, 3
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: s_add_i32 s26, s26, 3
-; GFX9-NEXT: v_writelane_b32 v30, s46, 48
-; GFX9-NEXT: s_lshr_b32 s46, s27, 16
-; GFX9-NEXT: s_lshr_b32 s83, s28, 8
-; GFX9-NEXT: s_lshr_b32 s81, s27, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 49
-; GFX9-NEXT: s_lshr_b32 s53, s27, 8
-; GFX9-NEXT: s_lshr_b32 s82, s26, 16
-; GFX9-NEXT: s_lshr_b32 s67, s26, 8
-; GFX9-NEXT: s_lshr_b32 s84, s25, 24
-; GFX9-NEXT: s_lshr_b32 s96, s25, 16
-; GFX9-NEXT: s_lshr_b32 s65, s25, 8
-; GFX9-NEXT: s_lshr_b32 s86, s24, 16
-; GFX9-NEXT: s_lshr_b32 s55, s24, 8
-; GFX9-NEXT: s_lshr_b32 s98, s23, 24
-; GFX9-NEXT: s_lshr_b32 s87, s23, 16
-; GFX9-NEXT: s_lshr_b32 s66, s23, 8
-; GFX9-NEXT: s_lshr_b32 s97, s22, 16
-; GFX9-NEXT: s_lshr_b32 s64, s22, 8
-; GFX9-NEXT: s_lshr_b32 s49, s21, 24
-; GFX9-NEXT: s_lshr_b32 s99, s21, 16
-; GFX9-NEXT: s_lshr_b32 s80, s21, 8
-; GFX9-NEXT: s_lshr_b32 s39, s20, 16
-; GFX9-NEXT: s_lshr_b32 s70, s20, 8
-; GFX9-NEXT: s_lshr_b32 s48, s19, 24
-; GFX9-NEXT: s_lshr_b32 s38, s19, 16
-; GFX9-NEXT: s_lshr_b32 s68, s19, 8
-; GFX9-NEXT: s_lshr_b32 s54, s18, 16
-; GFX9-NEXT: s_lshr_b32 s71, s18, 8
-; GFX9-NEXT: s_lshr_b32 s52, s17, 24
-; GFX9-NEXT: s_lshr_b32 s50, s17, 16
-; GFX9-NEXT: s_lshr_b32 s69, s17, 8
-; GFX9-NEXT: s_lshr_b32 s51, s16, 16
-; GFX9-NEXT: s_lshr_b32 s85, s16, 8
-; GFX9-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; GFX9-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; GFX9-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; GFX9-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; GFX9-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; GFX9-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; GFX9-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 4
+; GFX9-NEXT: v_writelane_b32 v30, s93, 5
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[28:29], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 2
+; GFX9-NEXT: v_writelane_b32 v30, s93, 3
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[26:27], 24
+; GFX9-NEXT: s_add_i32 s7, s7, 3
+; GFX9-NEXT: s_add_i32 s6, s6, 3
+; GFX9-NEXT: v_writelane_b32 v30, s92, 0
+; GFX9-NEXT: s_lshr_b32 s36, s5, 24
+; GFX9-NEXT: s_lshr_b32 s37, s5, 16
+; GFX9-NEXT: s_lshr_b32 s82, s5, 8
+; GFX9-NEXT: s_lshr_b32 s81, s4, 16
+; GFX9-NEXT: s_lshr_b32 s83, s4, 8
+; GFX9-NEXT: s_lshr_b32 s86, s7, 24
+; GFX9-NEXT: s_lshr_b32 s84, s7, 16
+; GFX9-NEXT: s_lshr_b32 s87, s7, 8
+; GFX9-NEXT: s_lshr_b32 s96, s6, 16
+; GFX9-NEXT: s_lshr_b32 s97, s6, 8
+; GFX9-NEXT: s_lshr_b32 s98, s9, 24
+; GFX9-NEXT: s_lshr_b32 s46, s9, 16
+; GFX9-NEXT: s_lshr_b32 s47, s9, 8
+; GFX9-NEXT: s_lshr_b32 s39, s8, 16
+; GFX9-NEXT: s_lshr_b32 s38, s8, 8
+; GFX9-NEXT: s_lshr_b32 s57, s11, 24
+; GFX9-NEXT: s_lshr_b32 s56, s11, 16
+; GFX9-NEXT: s_lshr_b32 s49, s11, 8
+; GFX9-NEXT: s_lshr_b32 s48, s10, 16
+; GFX9-NEXT: s_lshr_b32 s50, s10, 8
+; GFX9-NEXT: s_lshr_b32 s51, s13, 24
+; GFX9-NEXT: s_lshr_b32 s52, s13, 16
+; GFX9-NEXT: s_lshr_b32 s59, s13, 8
+; GFX9-NEXT: s_lshr_b32 s58, s12, 16
+; GFX9-NEXT: s_lshr_b32 s67, s12, 8
+; GFX9-NEXT: s_lshr_b32 s60, s15, 24
+; GFX9-NEXT: s_lshr_b32 s65, s15, 16
+; GFX9-NEXT: s_lshr_b32 s61, s15, 8
+; GFX9-NEXT: s_lshr_b32 s62, s14, 16
+; GFX9-NEXT: s_lshr_b32 s55, s14, 8
+; GFX9-NEXT: s_lshr_b32 s63, s41, 24
+; GFX9-NEXT: s_lshr_b32 s72, s41, 16
+; GFX9-NEXT: s_lshr_b32 s66, s41, 8
+; GFX9-NEXT: s_lshr_b32 s73, s40, 16
+; GFX9-NEXT: s_lshr_b32 s64, s40, 8
+; GFX9-NEXT: s_lshr_b32 s74, s43, 24
+; GFX9-NEXT: s_lshr_b32 s80, s43, 16
+; GFX9-NEXT: s_lshr_b32 s75, s43, 8
+; GFX9-NEXT: s_lshr_b32 s76, s42, 16
+; GFX9-NEXT: s_lshr_b32 s70, s42, 8
+; GFX9-NEXT: s_lshr_b32 s77, s45, 24
+; GFX9-NEXT: s_lshr_b32 s78, s45, 16
+; GFX9-NEXT: s_lshr_b32 s68, s45, 8
+; GFX9-NEXT: s_lshr_b32 s79, s44, 16
+; GFX9-NEXT: s_lshr_b32 s71, s44, 8
+; GFX9-NEXT: s_lshr_b32 s69, s29, 24
+; GFX9-NEXT: s_lshr_b32 s85, s29, 16
+; GFX9-NEXT: s_lshr_b32 s88, s29, 8
+; GFX9-NEXT: s_lshr_b32 s54, s28, 16
+; GFX9-NEXT: s_lshr_b32 s89, s28, 8
+; GFX9-NEXT: s_lshr_b32 s99, s27, 24
+; GFX9-NEXT: s_lshr_b32 s53, s27, 16
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[6:7], 24
+; GFX9-NEXT: v_writelane_b32 v30, s93, 1
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: .LBB13_3: ; %end
-; GFX9-NEXT: v_mov_b32_e32 v7, s36
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: .LBB13_5: ; %end
+; GFX9-NEXT: v_readlane_b32 s91, v30, 50
+; GFX9-NEXT: v_mov_b32_e32 v2, s91
+; GFX9-NEXT: v_readlane_b32 s91, v30, 48
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s85
-; GFX9-NEXT: v_perm_b32 v7, s51, v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, s91
; GFX9-NEXT: v_perm_b32 v3, s16, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v21, s52
-; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v4, s69
-; GFX9-NEXT: v_perm_b32 v21, s50, v21, v1
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-NEXT: v_readlane_b32 s16, v30, 45
; GFX9-NEXT: v_perm_b32 v2, s17, v4, v1
-; GFX9-NEXT: v_mov_b32_e32 v11, s34
+; GFX9-NEXT: v_mov_b32_e32 v4, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 43
+; GFX9-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 40
+; GFX9-NEXT: v_perm_b32 v5, s18, v4, v1
+; GFX9-NEXT: v_perm_b32 v4, s19, v6, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 38
+; GFX9-NEXT: v_mov_b32_e32 v9, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 35
+; GFX9-NEXT: v_perm_b32 v7, s20, v6, v1
+; GFX9-NEXT: v_perm_b32 v6, s21, v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v9, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 33
+; GFX9-NEXT: v_mov_b32_e32 v12, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 30
+; GFX9-NEXT: v_perm_b32 v10, s22, v9, v1
+; GFX9-NEXT: v_perm_b32 v9, s23, v12, v1
+; GFX9-NEXT: v_mov_b32_e32 v12, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 28
+; GFX9-NEXT: v_mov_b32_e32 v15, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 0
+; GFX9-NEXT: v_mov_b32_e32 v19, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 25
+; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
+; GFX9-NEXT: v_perm_b32 v12, s25, v15, v1
+; GFX9-NEXT: v_readlane_b32 s17, v30, 1
+; GFX9-NEXT: v_mov_b32_e32 v15, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 23
+; GFX9-NEXT: v_mov_b32_e32 v20, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 2
+; GFX9-NEXT: v_mov_b32_e32 v8, s34
+; GFX9-NEXT: v_perm_b32 v16, s26, v15, v1
+; GFX9-NEXT: v_perm_b32 v15, s27, v20, v1
+; GFX9-NEXT: v_mov_b32_e32 v20, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 49
+; GFX9-NEXT: v_perm_b32 v8, s16, v8, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 46
+; GFX9-NEXT: v_mov_b32_e32 v21, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 47
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v21, s16, v21, v1
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX9-NEXT: v_mov_b32_e32 v11, s30
+; GFX9-NEXT: v_readlane_b32 s16, v30, 44
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v21
-; GFX9-NEXT: v_mov_b32_e32 v4, s71
-; GFX9-NEXT: v_perm_b32 v11, s54, v11, v1
+; GFX9-NEXT: v_perm_b32 v11, s16, v11, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 41
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_perm_b32 v5, s18, v4, v1
-; GFX9-NEXT: v_mov_b32_e32 v22, s48
+; GFX9-NEXT: v_mov_b32_e32 v22, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 42
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v11
-; GFX9-NEXT: v_mov_b32_e32 v6, s68
-; GFX9-NEXT: v_perm_b32 v22, s38, v22, v1
+; GFX9-NEXT: v_perm_b32 v22, s16, v22, v1
; GFX9-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX9-NEXT: v_perm_b32 v4, s19, v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v14, s30
+; GFX9-NEXT: v_mov_b32_e32 v14, vcc_lo
+; GFX9-NEXT: v_readlane_b32 s16, v30, 39
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; GFX9-NEXT: v_mov_b32_e32 v6, s70
-; GFX9-NEXT: v_perm_b32 v14, s39, v14, v1
+; GFX9-NEXT: v_perm_b32 v14, s16, v14, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 36
; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX9-NEXT: v_perm_b32 v8, s20, v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v23, s49
+; GFX9-NEXT: v_mov_b32_e32 v23, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 37
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v14
-; GFX9-NEXT: v_mov_b32_e32 v9, s80
-; GFX9-NEXT: v_perm_b32 v23, s99, v23, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v8, v2
-; GFX9-NEXT: v_perm_b32 v6, s21, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v15, s94
+; GFX9-NEXT: v_perm_b32 v23, s16, v23, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX9-NEXT: v_mov_b32_e32 v17, s94
+; GFX9-NEXT: v_readlane_b32 s16, v30, 34
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v23
-; GFX9-NEXT: v_mov_b32_e32 v9, s64
-; GFX9-NEXT: v_perm_b32 v15, s97, v15, v1
+; GFX9-NEXT: v_perm_b32 v17, s16, v17, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 31
; GFX9-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX9-NEXT: v_perm_b32 v10, s22, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v24, s98
+; GFX9-NEXT: v_mov_b32_e32 v24, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 32
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v15
-; GFX9-NEXT: v_mov_b32_e32 v12, s66
-; GFX9-NEXT: v_perm_b32 v24, s87, v24, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX9-NEXT: v_perm_b32 v24, s16, v24, v1
; GFX9-NEXT: v_or_b32_e32 v2, v10, v2
-; GFX9-NEXT: v_perm_b32 v9, s23, v12, v1
-; GFX9-NEXT: v_mov_b32_e32 v16, s92
+; GFX9-NEXT: v_mov_b32_e32 v18, s92
+; GFX9-NEXT: v_readlane_b32 s16, v30, 29
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v24
-; GFX9-NEXT: v_mov_b32_e32 v12, s55
-; GFX9-NEXT: v_perm_b32 v16, s86, v16, v1
+; GFX9-NEXT: v_perm_b32 v18, s16, v18, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 26
; GFX9-NEXT: v_or_b32_e32 v2, v9, v2
-; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
-; GFX9-NEXT: v_mov_b32_e32 v25, s84
+; GFX9-NEXT: v_mov_b32_e32 v25, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 27
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v16
-; GFX9-NEXT: v_mov_b32_e32 v17, s65
-; GFX9-NEXT: v_perm_b32 v25, s96, v25, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v18
+; GFX9-NEXT: v_perm_b32 v25, s16, v25, v1
; GFX9-NEXT: v_or_b32_e32 v2, v13, v2
-; GFX9-NEXT: v_perm_b32 v12, s25, v17, v1
-; GFX9-NEXT: v_mov_b32_e32 v17, s90
+; GFX9-NEXT: v_readlane_b32 s16, v30, 24
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:32
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; GFX9-NEXT: v_mov_b32_e32 v18, s67
-; GFX9-NEXT: v_perm_b32 v17, s82, v17, v1
+; GFX9-NEXT: v_perm_b32 v19, s16, v19, v1
; GFX9-NEXT: v_or_b32_e32 v2, v12, v2
-; GFX9-NEXT: v_perm_b32 v18, s26, v18, v1
-; GFX9-NEXT: v_mov_b32_e32 v26, s81
-; GFX9-NEXT: v_readlane_b32 s16, v30, 49
+; GFX9-NEXT: v_mov_b32_e32 v26, s99
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v17
-; GFX9-NEXT: v_mov_b32_e32 v19, s53
-; GFX9-NEXT: v_perm_b32 v26, s16, v26, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v18, v2
-; GFX9-NEXT: v_perm_b32 v19, s27, v19, v1
-; GFX9-NEXT: v_mov_b32_e32 v20, s88
-; GFX9-NEXT: v_readlane_b32 s16, v30, 48
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v19
+; GFX9-NEXT: v_perm_b32 v26, s53, v26, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v16, v2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v26
-; GFX9-NEXT: v_mov_b32_e32 v27, s83
-; GFX9-NEXT: v_perm_b32 v20, s16, v20, v1
-; GFX9-NEXT: v_readlane_b32 s16, v30, 47
-; GFX9-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX9-NEXT: v_mov_b32_e32 v28, s16
+; GFX9-NEXT: v_mov_b32_e32 v27, s89
+; GFX9-NEXT: v_perm_b32 v20, s54, v20, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v15, v2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: v_perm_b32 v2, s28, v27, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v20
-; GFX9-NEXT: v_readlane_b32 s16, v30, 45
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 46
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, s69
+; GFX9-NEXT: v_readlane_b32 s17, v30, 3
+; GFX9-NEXT: v_mov_b32_e32 v28, s88
+; GFX9-NEXT: v_perm_b32 v3, s85, v3, v1
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
; GFX9-NEXT: v_perm_b32 v2, s29, v28, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 4
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 44
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s78
-; GFX9-NEXT: v_readlane_b32 s16, v30, 43
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s71
+; GFX9-NEXT: v_perm_b32 v3, s79, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s44, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 42
+; GFX9-NEXT: v_mov_b32_e32 v3, s77
+; GFX9-NEXT: v_readlane_b32 s17, v30, 5
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 40
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 41
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s68
+; GFX9-NEXT: v_perm_b32 v3, s78, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s45, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 6
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 39
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:60
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s76
-; GFX9-NEXT: v_readlane_b32 s16, v30, 38
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s70
+; GFX9-NEXT: v_perm_b32 v3, s76, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s42, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 37
+; GFX9-NEXT: v_mov_b32_e32 v3, s74
+; GFX9-NEXT: v_readlane_b32 s17, v30, 7
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 35
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 36
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s75
+; GFX9-NEXT: v_perm_b32 v3, s80, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s43, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 8
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 34
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s74
-; GFX9-NEXT: v_readlane_b32 s16, v30, 33
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s64
+; GFX9-NEXT: v_perm_b32 v3, s73, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s40, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 32
+; GFX9-NEXT: v_mov_b32_e32 v3, s63
+; GFX9-NEXT: v_readlane_b32 s17, v30, 9
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 30
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 31
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s66
+; GFX9-NEXT: v_perm_b32 v3, s72, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s41, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 10
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 29
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
+; GFX9-NEXT: v_mov_b32_e32 v2, s55
+; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s14, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s72
-; GFX9-NEXT: v_readlane_b32 s14, v30, 28
-; GFX9-NEXT: v_perm_b32 v3, s14, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 27
+; GFX9-NEXT: v_mov_b32_e32 v3, s60
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:80
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_readlane_b32 s14, v30, 25
-; GFX9-NEXT: v_mov_b32_e32 v3, s14
-; GFX9-NEXT: v_readlane_b32 s14, v30, 26
-; GFX9-NEXT: v_perm_b32 v3, s14, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s61
+; GFX9-NEXT: v_perm_b32 v3, s65, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s15, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s14, v30, 12
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 24
+; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
+; GFX9-NEXT: v_mov_b32_e32 v2, s67
+; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s12, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s62
-; GFX9-NEXT: v_readlane_b32 s12, v30, 23
-; GFX9-NEXT: v_perm_b32 v3, s12, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 22
+; GFX9-NEXT: v_mov_b32_e32 v3, s51
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:88
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_readlane_b32 s12, v30, 20
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
-; GFX9-NEXT: v_readlane_b32 s12, v30, 21
-; GFX9-NEXT: v_perm_b32 v3, s12, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s59
+; GFX9-NEXT: v_perm_b32 v3, s52, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s13, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s12, v30, 14
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 19
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
+; GFX9-NEXT: v_mov_b32_e32 v2, s50
+; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s10, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s60
-; GFX9-NEXT: v_readlane_b32 s10, v30, 18
-; GFX9-NEXT: v_perm_b32 v3, s10, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 17
+; GFX9-NEXT: v_mov_b32_e32 v3, s57
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:96
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_readlane_b32 s10, v30, 15
-; GFX9-NEXT: v_mov_b32_e32 v3, s10
-; GFX9-NEXT: v_readlane_b32 s10, v30, 16
-; GFX9-NEXT: v_perm_b32 v3, s10, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s49
+; GFX9-NEXT: v_perm_b32 v3, s56, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s11, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s10, v30, 16
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 14
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v2, s38
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s8, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s58
-; GFX9-NEXT: v_readlane_b32 s8, v30, 13
-; GFX9-NEXT: v_perm_b32 v3, s8, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s8, v30, 12
+; GFX9-NEXT: v_mov_b32_e32 v3, s98
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:104
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NEXT: v_readlane_b32 s8, v30, 10
-; GFX9-NEXT: v_mov_b32_e32 v3, s8
-; GFX9-NEXT: v_readlane_b32 s8, v30, 11
-; GFX9-NEXT: v_perm_b32 v3, s8, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s47
+; GFX9-NEXT: v_perm_b32 v3, s46, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s9, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s8, v30, 9
+; GFX9-NEXT: v_mov_b32_e32 v3, s90
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-NEXT: v_mov_b32_e32 v2, s97
+; GFX9-NEXT: v_perm_b32 v3, s96, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s6, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s56
-; GFX9-NEXT: v_readlane_b32 s6, v30, 8
-; GFX9-NEXT: v_perm_b32 v3, s6, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 7
+; GFX9-NEXT: v_mov_b32_e32 v3, s86
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:112
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_readlane_b32 s6, v30, 5
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: v_readlane_b32 s6, v30, 6
-; GFX9-NEXT: v_perm_b32 v3, s6, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s87
+; GFX9-NEXT: v_perm_b32 v3, s84, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s7, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s6, v30, 18
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 4
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s83
+; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s46
-; GFX9-NEXT: v_readlane_b32 s4, v30, 3
-; GFX9-NEXT: v_perm_b32 v3, s4, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_readlane_b32 s4, v30, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: v_readlane_b32 s4, v30, 1
+; GFX9-NEXT: v_mov_b32_e32 v2, s82
+; GFX9-NEXT: v_mov_b32_e32 v3, s36
; GFX9-NEXT: v_perm_b32 v2, s5, v2, v1
-; GFX9-NEXT: v_perm_b32 v1, s4, v3, v1
+; GFX9-NEXT: v_perm_b32 v1, s37, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
; GFX9-NEXT: v_readlane_b32 s30, v29, 34
+; GFX9-NEXT: v_readlane_b32 s17, v30, 11
+; GFX9-NEXT: v_readlane_b32 s15, v30, 13
+; GFX9-NEXT: v_readlane_b32 s13, v30, 15
+; GFX9-NEXT: v_readlane_b32 s11, v30, 17
+; GFX9-NEXT: v_readlane_b32 s7, v30, 19
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s31, v29, 35
; GFX9-NEXT: v_readlane_b32 s99, v29, 33
@@ -10693,154 +10973,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v32i32_to_v128i8_scalar:
; GFX11: ; %bb.0:
@@ -10909,570 +11041,702 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s103, 0
; GFX11-NEXT: ; implicit-def: $vgpr26 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr25 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s42, s5, 24
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s27, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s5, 16
-; GFX11-NEXT: s_lshr_b32 s34, s27, 16
-; GFX11-NEXT: s_lshr_b32 s35, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s26, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s5, 8
-; GFX11-NEXT: s_lshr_b32 s36, s26, 8
-; GFX11-NEXT: s_lshr_b32 s39, s25, 24
-; GFX11-NEXT: s_lshr_b32 s48, s25, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 2
+; GFX11-NEXT: s_lshr_b32 s43, s8, 16
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[4:5], 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 18
+; GFX11-NEXT: s_lshr_b32 s43, s12, 16
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[8:9], 24
+; GFX11-NEXT: s_lshr_b32 s31, s5, 24
+; GFX11-NEXT: s_lshr_b32 s104, s5, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 19
+; GFX11-NEXT: s_lshr_b32 s43, s22, 16
+; GFX11-NEXT: s_lshr_b32 s30, s5, 8
; GFX11-NEXT: s_lshr_b32 s42, s4, 16
-; GFX11-NEXT: s_lshr_b32 s38, s25, 8
-; GFX11-NEXT: s_lshr_b32 s50, s24, 16
-; GFX11-NEXT: s_lshr_b32 s49, s24, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s4, 8
-; GFX11-NEXT: s_lshr_b32 s52, s23, 24
-; GFX11-NEXT: s_lshr_b32 s53, s23, 16
-; GFX11-NEXT: s_lshr_b32 s51, s23, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s7, 24
-; GFX11-NEXT: s_lshr_b32 s69, s22, 16
-; GFX11-NEXT: s_lshr_b32 s54, s22, 8
-; GFX11-NEXT: s_lshr_b32 s55, s21, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s7, 16
-; GFX11-NEXT: s_lshr_b32 s64, s21, 16
-; GFX11-NEXT: s_lshr_b32 s65, s21, 8
-; GFX11-NEXT: s_lshr_b32 s82, s20, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s7, 8
-; GFX11-NEXT: s_lshr_b32 s68, s20, 8
-; GFX11-NEXT: s_lshr_b32 s66, s19, 24
-; GFX11-NEXT: s_lshr_b32 s67, s19, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 7
-; GFX11-NEXT: s_lshr_b32 s42, s6, 16
-; GFX11-NEXT: s_lshr_b32 s85, s19, 8
-; GFX11-NEXT: s_lshr_b32 s86, s18, 16
-; GFX11-NEXT: s_lshr_b32 s87, s18, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 8
-; GFX11-NEXT: s_lshr_b32 s42, s6, 8
-; GFX11-NEXT: s_lshr_b32 s70, s17, 24
-; GFX11-NEXT: s_lshr_b32 s71, s17, 16
-; GFX11-NEXT: s_lshr_b32 s96, s17, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 9
-; GFX11-NEXT: s_lshr_b32 s42, s9, 24
-; GFX11-NEXT: s_lshr_b32 s97, s16, 16
-; GFX11-NEXT: s_lshr_b32 s98, s16, 8
-; GFX11-NEXT: s_lshr_b32 s80, s3, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 10
-; GFX11-NEXT: s_lshr_b32 s42, s9, 16
-; GFX11-NEXT: s_lshr_b32 s81, s3, 16
-; GFX11-NEXT: s_lshr_b32 s99, s3, 8
-; GFX11-NEXT: s_lshr_b32 s100, s2, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 11
-; GFX11-NEXT: s_lshr_b32 s42, s9, 8
-; GFX11-NEXT: s_lshr_b32 s101, s2, 8
-; GFX11-NEXT: s_lshr_b32 s83, s1, 24
-; GFX11-NEXT: s_lshr_b32 s84, s1, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 12
-; GFX11-NEXT: s_lshr_b32 s42, s8, 16
-; GFX11-NEXT: s_lshr_b32 s102, s1, 8
-; GFX11-NEXT: s_lshr_b32 s103, s0, 16
-; GFX11-NEXT: s_lshr_b32 s104, s0, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 13
-; GFX11-NEXT: s_lshr_b32 s42, s8, 8
-; GFX11-NEXT: s_lshr_b64 s[44:45], s[6:7], 24
-; GFX11-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; GFX11-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 14
-; GFX11-NEXT: s_lshr_b32 s42, s11, 24
-; GFX11-NEXT: s_lshr_b64 s[58:59], s[12:13], 24
-; GFX11-NEXT: s_lshr_b64 s[60:61], s[14:15], 24
-; GFX11-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 15
-; GFX11-NEXT: s_lshr_b32 s42, s11, 16
-; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
-; GFX11-NEXT: s_lshr_b64 s[76:77], s[26:27], 24
-; GFX11-NEXT: s_lshr_b64 s[72:73], s[24:25], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 16
-; GFX11-NEXT: s_lshr_b32 s42, s11, 8
-; GFX11-NEXT: s_lshr_b64 s[78:79], s[22:23], 24
+; GFX11-NEXT: s_lshr_b32 s34, s4, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 20
+; GFX11-NEXT: s_lshr_b32 s43, s20, 16
+; GFX11-NEXT: s_lshr_b32 s35, s7, 24
+; GFX11-NEXT: s_lshr_b32 s37, s7, 16
+; GFX11-NEXT: s_lshr_b32 s36, s7, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 21
+; GFX11-NEXT: s_lshr_b32 s43, s20, 8
+; GFX11-NEXT: s_lshr_b32 s61, s6, 16
+; GFX11-NEXT: s_lshr_b32 s38, s6, 8
+; GFX11-NEXT: s_lshr_b32 s48, s9, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 22
+; GFX11-NEXT: s_lshr_b32 s43, s19, 8
+; GFX11-NEXT: s_lshr_b32 s39, s9, 16
+; GFX11-NEXT: s_lshr_b32 s45, s9, 8
+; GFX11-NEXT: s_lshr_b32 s44, s8, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 23
+; GFX11-NEXT: s_lshr_b32 s43, s18, 16
+; GFX11-NEXT: s_lshr_b32 s47, s11, 24
+; GFX11-NEXT: s_lshr_b32 s49, s11, 16
+; GFX11-NEXT: s_lshr_b32 s46, s11, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 24
+; GFX11-NEXT: s_lshr_b32 s43, s18, 8
+; GFX11-NEXT: s_lshr_b32 s51, s10, 16
+; GFX11-NEXT: s_lshr_b32 s50, s10, 8
+; GFX11-NEXT: s_lshr_b32 s57, s13, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 25
+; GFX11-NEXT: s_lshr_b32 s43, s17, 24
+; GFX11-NEXT: s_lshr_b32 s56, s13, 16
+; GFX11-NEXT: s_lshr_b32 s59, s13, 8
+; GFX11-NEXT: s_lshr_b32 s58, s12, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 26
+; GFX11-NEXT: s_lshr_b32 s43, s17, 16
+; GFX11-NEXT: s_lshr_b32 s60, s15, 24
+; GFX11-NEXT: s_lshr_b32 s53, s15, 16
+; GFX11-NEXT: s_lshr_b32 s52, s15, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 27
+; GFX11-NEXT: s_lshr_b32 s43, s17, 8
+; GFX11-NEXT: s_lshr_b32 s62, s14, 16
+; GFX11-NEXT: s_lshr_b32 s63, s14, 8
+; GFX11-NEXT: s_lshr_b32 s54, s41, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 28
+; GFX11-NEXT: s_lshr_b32 s43, s16, 16
+; GFX11-NEXT: s_lshr_b32 s55, s41, 16
+; GFX11-NEXT: s_lshr_b32 s64, s41, 8
+; GFX11-NEXT: s_lshr_b32 s66, s40, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 29
+; GFX11-NEXT: s_lshr_b32 s43, s16, 8
+; GFX11-NEXT: s_lshr_b32 s65, s40, 8
+; GFX11-NEXT: s_lshr_b32 s69, s29, 24
+; GFX11-NEXT: s_lshr_b32 s71, s29, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 30
+; GFX11-NEXT: s_lshr_b32 s43, s3, 24
+; GFX11-NEXT: s_lshr_b32 s67, s29, 8
+; GFX11-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-NEXT: s_lshr_b32 s80, s28, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 31
+; GFX11-NEXT: s_lshr_b32 s43, s3, 16
+; GFX11-NEXT: s_lshr_b32 s82, s27, 24
+; GFX11-NEXT: v_writelane_b32 v25, s43, 0
+; GFX11-NEXT: s_lshr_b32 s43, s3, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
+; GFX11-NEXT: s_lshr_b32 s83, s27, 16
+; GFX11-NEXT: s_lshr_b32 s84, s27, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 1
+; GFX11-NEXT: s_lshr_b32 s43, s2, 16
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: s_lshr_b32 s86, s26, 16
+; GFX11-NEXT: s_lshr_b32 s85, s26, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 2
+; GFX11-NEXT: s_lshr_b32 s43, s2, 8
+; GFX11-NEXT: v_writelane_b32 v26, s74, 14
+; GFX11-NEXT: s_lshr_b32 s96, s25, 24
+; GFX11-NEXT: s_lshr_b32 s97, s25, 16
+; GFX11-NEXT: v_writelane_b32 v25, s43, 3
+; GFX11-NEXT: s_lshr_b32 s43, s1, 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 15
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[10:11], 24
+; GFX11-NEXT: s_lshr_b32 s87, s25, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 4
+; GFX11-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 12
+; GFX11-NEXT: s_lshr_b32 s99, s24, 16
+; GFX11-NEXT: s_lshr_b32 s98, s24, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 5
+; GFX11-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 13
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[12:13], 24
+; GFX11-NEXT: s_lshr_b32 s101, s23, 24
+; GFX11-NEXT: v_writelane_b32 v25, s43, 6
+; GFX11-NEXT: s_lshr_b32 s43, s0, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 10
+; GFX11-NEXT: s_lshr_b32 s102, s23, 16
+; GFX11-NEXT: s_lshr_b32 s100, s23, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 7
+; GFX11-NEXT: s_lshr_b32 s43, s0, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 11
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[14:15], 24
+; GFX11-NEXT: s_lshr_b32 s76, s22, 8
+; GFX11-NEXT: s_lshr_b32 s78, s21, 24
+; GFX11-NEXT: s_lshr_b32 s77, s21, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 8
+; GFX11-NEXT: s_lshr_b32 s68, s21, 8
+; GFX11-NEXT: s_lshr_b32 s79, s19, 24
+; GFX11-NEXT: s_lshr_b32 s70, s19, 16
+; GFX11-NEXT: v_writelane_b32 v25, s43, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 9
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[6:7], 24
; GFX11-NEXT: s_lshr_b64 s[88:89], s[20:21], 24
; GFX11-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 17
-; GFX11-NEXT: s_lshr_b32 s42, s10, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 6
; GFX11-NEXT: s_lshr_b64 s[92:93], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[94:95], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[0:1], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 18
-; GFX11-NEXT: s_lshr_b32 s42, s10, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 19
-; GFX11-NEXT: s_lshr_b32 s42, s13, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 20
-; GFX11-NEXT: s_lshr_b32 s42, s13, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 21
-; GFX11-NEXT: s_lshr_b32 s42, s13, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 22
-; GFX11-NEXT: s_lshr_b32 s42, s12, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 23
-; GFX11-NEXT: s_lshr_b32 s42, s12, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 24
-; GFX11-NEXT: s_lshr_b32 s42, s15, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 25
-; GFX11-NEXT: s_lshr_b32 s42, s15, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 26
-; GFX11-NEXT: s_lshr_b32 s42, s15, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 27
-; GFX11-NEXT: s_lshr_b32 s42, s14, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 28
-; GFX11-NEXT: s_lshr_b32 s42, s14, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 29
-; GFX11-NEXT: s_lshr_b32 s42, s41, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 30
-; GFX11-NEXT: s_lshr_b32 s42, s41, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 31
-; GFX11-NEXT: s_lshr_b32 s42, s41, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s40, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s40, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s29, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s29, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s29, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s28, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s28, 8
+; GFX11-NEXT: s_lshr_b64 vcc, s[0:1], 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 7
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 4
+; GFX11-NEXT: v_writelane_b32 v26, s75, 5
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[26:27], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 2
+; GFX11-NEXT: v_writelane_b32 v26, s75, 3
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[24:25], 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 7
-; GFX11-NEXT: s_lshr_b64 s[42:43], s[4:5], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: v_writelane_b32 v26, s74, 0
+; GFX11-NEXT: v_writelane_b32 v26, s75, 1
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[22:23], 24
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: s_mov_b32 s103, -1
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr70
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr68
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr100
+; GFX11-NEXT: ; implicit-def: $sgpr102
+; GFX11-NEXT: ; implicit-def: $sgpr101
+; GFX11-NEXT: ; implicit-def: $sgpr98
+; GFX11-NEXT: ; implicit-def: $sgpr99
+; GFX11-NEXT: ; implicit-def: $sgpr87
+; GFX11-NEXT: ; implicit-def: $sgpr97
+; GFX11-NEXT: ; implicit-def: $sgpr96
+; GFX11-NEXT: ; implicit-def: $sgpr85
+; GFX11-NEXT: ; implicit-def: $sgpr86
+; GFX11-NEXT: ; implicit-def: $sgpr84
+; GFX11-NEXT: ; implicit-def: $sgpr83
+; GFX11-NEXT: ; implicit-def: $sgpr82
+; GFX11-NEXT: ; implicit-def: $sgpr80
+; GFX11-NEXT: ; implicit-def: $sgpr81
+; GFX11-NEXT: ; implicit-def: $sgpr67
+; GFX11-NEXT: ; implicit-def: $sgpr71
+; GFX11-NEXT: ; implicit-def: $sgpr69
+; GFX11-NEXT: ; implicit-def: $sgpr65
+; GFX11-NEXT: ; implicit-def: $sgpr66
+; GFX11-NEXT: ; implicit-def: $sgpr64
+; GFX11-NEXT: ; implicit-def: $sgpr55
+; GFX11-NEXT: ; implicit-def: $sgpr54
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr52
+; GFX11-NEXT: ; implicit-def: $sgpr53
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr50
+; GFX11-NEXT: ; implicit-def: $sgpr51
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr49
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr104
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: v_writelane_b32 v26, s42, 0
+; GFX11-NEXT: v_writelane_b32 v26, s43, 1
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: v_writelane_b32 v26, s72, 2
+; GFX11-NEXT: v_writelane_b32 v26, s73, 3
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 4
+; GFX11-NEXT: v_writelane_b32 v26, s73, 5
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 6
+; GFX11-NEXT: v_writelane_b32 v26, s73, 7
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 8
+; GFX11-NEXT: v_writelane_b32 v26, s73, 9
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 10
+; GFX11-NEXT: v_writelane_b32 v26, s73, 11
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 12
+; GFX11-NEXT: v_writelane_b32 v26, s73, 13
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 14
+; GFX11-NEXT: v_writelane_b32 v26, s73, 15
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_and_b32 s43, s103, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_mov_b32 s103, s31
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_mov_b32 s31, s42
+; GFX11-NEXT: s_mov_b32 s42, s61
+; GFX11-NEXT: s_mov_b32 s43, s48
+; GFX11-NEXT: v_readlane_b32 s48, v26, 18
+; GFX11-NEXT: v_readlane_b32 s61, v26, 19
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_lshr_b32 s72, s22, 16
+; GFX11-NEXT: s_add_i32 s19, s19, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 20
+; GFX11-NEXT: s_lshr_b32 s72, s20, 16
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s17, s17, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 21
+; GFX11-NEXT: s_lshr_b32 s72, s20, 8
+; GFX11-NEXT: s_add_i32 s3, s3, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s1, s1, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 22
+; GFX11-NEXT: s_lshr_b32 s72, s19, 8
+; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s4, s4, 3
-; GFX11-NEXT: s_lshr_b32 s42, s5, 24
-; GFX11-NEXT: s_add_i32 s7, s7, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s5, 16
-; GFX11-NEXT: s_add_i32 s6, s6, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 23
+; GFX11-NEXT: s_lshr_b32 s72, s18, 16
; GFX11-NEXT: s_add_i32 s9, s9, 3
; GFX11-NEXT: s_add_i32 s8, s8, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s5, 8
; GFX11-NEXT: s_add_i32 s11, s11, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 24
+; GFX11-NEXT: s_lshr_b32 s72, s18, 8
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[8:9], 24
; GFX11-NEXT: s_add_i32 s10, s10, 3
; GFX11-NEXT: s_add_i32 s13, s13, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s4, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 25
+; GFX11-NEXT: s_lshr_b32 s72, s17, 24
; GFX11-NEXT: s_add_i32 s12, s12, 3
; GFX11-NEXT: s_add_i32 s15, s15, 3
; GFX11-NEXT: s_add_i32 s14, s14, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s4, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 26
+; GFX11-NEXT: s_lshr_b32 s72, s17, 16
; GFX11-NEXT: s_add_i32 s41, s41, 3
; GFX11-NEXT: s_add_i32 s40, s40, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s7, 24
+; GFX11-NEXT: v_writelane_b32 v26, s72, 27
+; GFX11-NEXT: s_lshr_b32 s72, s17, 8
; GFX11-NEXT: s_add_i32 s28, s28, 3
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s7, 16
-; GFX11-NEXT: s_add_i32 s3, s3, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s17, s17, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s7, 8
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s19, s19, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 7
-; GFX11-NEXT: s_lshr_b32 s42, s6, 16
-; GFX11-NEXT: s_add_i32 s21, s21, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s23, s23, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 8
-; GFX11-NEXT: s_lshr_b32 s42, s6, 8
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s25, s25, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: v_writelane_b32 v26, s42, 9
-; GFX11-NEXT: s_lshr_b32 s42, s9, 24
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s27, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 10
-; GFX11-NEXT: s_lshr_b32 s42, s9, 16
-; GFX11-NEXT: s_lshr_b32 s34, s27, 16
-; GFX11-NEXT: s_lshr_b32 s35, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s26, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 11
-; GFX11-NEXT: s_lshr_b32 s42, s9, 8
-; GFX11-NEXT: s_lshr_b32 s36, s26, 8
-; GFX11-NEXT: s_lshr_b32 s39, s25, 24
-; GFX11-NEXT: s_lshr_b32 s48, s25, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 12
-; GFX11-NEXT: s_lshr_b32 s42, s8, 16
-; GFX11-NEXT: s_lshr_b32 s38, s25, 8
-; GFX11-NEXT: s_lshr_b32 s50, s24, 16
-; GFX11-NEXT: s_lshr_b32 s49, s24, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 13
-; GFX11-NEXT: s_lshr_b32 s42, s8, 8
-; GFX11-NEXT: s_lshr_b32 s52, s23, 24
-; GFX11-NEXT: s_lshr_b32 s53, s23, 16
-; GFX11-NEXT: s_lshr_b32 s51, s23, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 14
-; GFX11-NEXT: s_lshr_b32 s42, s11, 24
-; GFX11-NEXT: s_lshr_b32 s69, s22, 16
-; GFX11-NEXT: s_lshr_b32 s54, s22, 8
-; GFX11-NEXT: s_lshr_b32 s55, s21, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 15
-; GFX11-NEXT: s_lshr_b32 s42, s11, 16
-; GFX11-NEXT: s_lshr_b32 s64, s21, 16
-; GFX11-NEXT: s_lshr_b32 s65, s21, 8
-; GFX11-NEXT: s_lshr_b32 s82, s20, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 16
-; GFX11-NEXT: s_lshr_b32 s42, s11, 8
-; GFX11-NEXT: s_lshr_b32 s68, s20, 8
-; GFX11-NEXT: s_lshr_b32 s66, s19, 24
-; GFX11-NEXT: s_lshr_b32 s67, s19, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 17
-; GFX11-NEXT: s_lshr_b32 s42, s10, 16
-; GFX11-NEXT: s_lshr_b32 s85, s19, 8
-; GFX11-NEXT: s_lshr_b32 s86, s18, 16
-; GFX11-NEXT: s_lshr_b32 s87, s18, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 18
-; GFX11-NEXT: s_lshr_b32 s42, s10, 8
-; GFX11-NEXT: s_lshr_b32 s70, s17, 24
-; GFX11-NEXT: s_lshr_b32 s71, s17, 16
-; GFX11-NEXT: s_lshr_b32 s96, s17, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 19
-; GFX11-NEXT: s_lshr_b32 s42, s13, 24
-; GFX11-NEXT: s_lshr_b32 s97, s16, 16
-; GFX11-NEXT: s_lshr_b32 s98, s16, 8
-; GFX11-NEXT: s_lshr_b32 s80, s3, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 20
-; GFX11-NEXT: s_lshr_b32 s42, s13, 16
-; GFX11-NEXT: s_lshr_b32 s81, s3, 16
-; GFX11-NEXT: s_lshr_b32 s99, s3, 8
-; GFX11-NEXT: s_lshr_b32 s100, s2, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 21
-; GFX11-NEXT: s_lshr_b32 s42, s13, 8
-; GFX11-NEXT: s_lshr_b32 s101, s2, 8
-; GFX11-NEXT: s_lshr_b32 s83, s1, 24
-; GFX11-NEXT: s_lshr_b32 s84, s1, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 22
-; GFX11-NEXT: s_lshr_b32 s42, s12, 16
-; GFX11-NEXT: s_lshr_b32 s102, s1, 8
-; GFX11-NEXT: s_lshr_b32 s103, s0, 16
-; GFX11-NEXT: s_lshr_b32 s104, s0, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 23
-; GFX11-NEXT: s_lshr_b32 s42, s12, 8
-; GFX11-NEXT: s_lshr_b64 s[44:45], s[6:7], 24
-; GFX11-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; GFX11-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 24
-; GFX11-NEXT: s_lshr_b32 s42, s15, 24
-; GFX11-NEXT: s_lshr_b64 s[58:59], s[12:13], 24
-; GFX11-NEXT: s_lshr_b64 s[60:61], s[14:15], 24
-; GFX11-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 25
-; GFX11-NEXT: s_lshr_b32 s42, s15, 16
-; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
-; GFX11-NEXT: s_lshr_b64 s[76:77], s[26:27], 24
-; GFX11-NEXT: s_lshr_b64 s[72:73], s[24:25], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 26
-; GFX11-NEXT: s_lshr_b32 s42, s15, 8
-; GFX11-NEXT: s_lshr_b64 s[78:79], s[22:23], 24
+; GFX11-NEXT: v_writelane_b32 v26, s72, 28
+; GFX11-NEXT: s_lshr_b32 s72, s16, 16
+; GFX11-NEXT: s_add_i32 s25, s25, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s21, s21, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 29
+; GFX11-NEXT: s_lshr_b32 s72, s16, 8
+; GFX11-NEXT: s_add_i32 s23, s23, 3
+; GFX11-NEXT: s_add_i32 s7, s7, 3
+; GFX11-NEXT: s_add_i32 s6, s6, 3
+; GFX11-NEXT: v_writelane_b32 v26, s72, 30
+; GFX11-NEXT: s_lshr_b32 s72, s3, 24
+; GFX11-NEXT: s_lshr_b32 s103, s5, 24
+; GFX11-NEXT: s_lshr_b32 s104, s5, 16
+; GFX11-NEXT: s_lshr_b32 s30, s5, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 31
+; GFX11-NEXT: s_lshr_b32 s72, s3, 16
+; GFX11-NEXT: s_lshr_b32 s31, s4, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 0
+; GFX11-NEXT: s_lshr_b32 s72, s3, 8
+; GFX11-NEXT: s_lshr_b32 s34, s4, 8
+; GFX11-NEXT: s_lshr_b32 s35, s7, 24
+; GFX11-NEXT: s_lshr_b32 s37, s7, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 1
+; GFX11-NEXT: s_lshr_b32 s72, s2, 16
+; GFX11-NEXT: s_lshr_b32 s36, s7, 8
+; GFX11-NEXT: s_lshr_b32 s42, s6, 16
+; GFX11-NEXT: s_lshr_b32 s38, s6, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 2
+; GFX11-NEXT: s_lshr_b32 s72, s2, 8
+; GFX11-NEXT: s_lshr_b32 s43, s9, 24
+; GFX11-NEXT: s_lshr_b32 s39, s9, 16
+; GFX11-NEXT: s_lshr_b32 s45, s9, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 3
+; GFX11-NEXT: s_lshr_b32 s72, s1, 24
+; GFX11-NEXT: s_lshr_b32 s48, s8, 16
+; GFX11-NEXT: s_lshr_b32 s44, s8, 8
+; GFX11-NEXT: s_lshr_b32 s47, s11, 24
+; GFX11-NEXT: v_writelane_b32 v25, s72, 4
+; GFX11-NEXT: s_lshr_b32 s72, s1, 16
+; GFX11-NEXT: s_lshr_b32 s49, s11, 16
+; GFX11-NEXT: s_lshr_b32 s46, s11, 8
+; GFX11-NEXT: s_lshr_b32 s51, s10, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 5
+; GFX11-NEXT: s_lshr_b32 s72, s1, 8
+; GFX11-NEXT: s_lshr_b32 s50, s10, 8
+; GFX11-NEXT: s_lshr_b32 s57, s13, 24
+; GFX11-NEXT: s_lshr_b32 s56, s13, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 6
+; GFX11-NEXT: s_lshr_b32 s72, s0, 16
+; GFX11-NEXT: s_lshr_b32 s59, s13, 8
+; GFX11-NEXT: s_lshr_b32 s61, s12, 16
+; GFX11-NEXT: s_lshr_b32 s58, s12, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 7
+; GFX11-NEXT: s_lshr_b32 s72, s0, 8
+; GFX11-NEXT: s_lshr_b32 s60, s15, 24
+; GFX11-NEXT: s_lshr_b32 s53, s15, 16
+; GFX11-NEXT: s_lshr_b32 s52, s15, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 8
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[4:5], 24
+; GFX11-NEXT: s_lshr_b32 s62, s14, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
+; GFX11-NEXT: s_lshr_b32 s63, s14, 8
+; GFX11-NEXT: s_lshr_b32 s54, s41, 24
+; GFX11-NEXT: s_lshr_b32 s55, s41, 16
+; GFX11-NEXT: s_lshr_b32 s64, s41, 8
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: s_lshr_b32 s66, s40, 16
+; GFX11-NEXT: s_lshr_b32 s65, s40, 8
+; GFX11-NEXT: s_lshr_b32 s69, s29, 24
+; GFX11-NEXT: s_lshr_b32 s71, s29, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 14
+; GFX11-NEXT: s_lshr_b32 s67, s29, 8
+; GFX11-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-NEXT: s_lshr_b32 s80, s28, 8
+; GFX11-NEXT: s_lshr_b32 s82, s27, 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 15
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[10:11], 24
+; GFX11-NEXT: s_lshr_b32 s83, s27, 16
+; GFX11-NEXT: s_lshr_b32 s84, s27, 8
+; GFX11-NEXT: s_lshr_b32 s86, s26, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 12
+; GFX11-NEXT: s_lshr_b32 s85, s26, 8
+; GFX11-NEXT: s_lshr_b32 s96, s25, 24
+; GFX11-NEXT: s_lshr_b32 s97, s25, 16
+; GFX11-NEXT: s_lshr_b32 s87, s25, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 13
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[12:13], 24
+; GFX11-NEXT: s_lshr_b32 s99, s24, 16
+; GFX11-NEXT: s_lshr_b32 s98, s24, 8
+; GFX11-NEXT: s_lshr_b32 s101, s23, 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 10
+; GFX11-NEXT: s_lshr_b32 s102, s23, 16
+; GFX11-NEXT: s_lshr_b32 s100, s23, 8
+; GFX11-NEXT: s_lshr_b32 s76, s22, 8
+; GFX11-NEXT: s_lshr_b32 s78, s21, 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 11
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[14:15], 24
+; GFX11-NEXT: s_lshr_b32 s77, s21, 16
+; GFX11-NEXT: s_lshr_b32 s68, s21, 8
+; GFX11-NEXT: s_lshr_b32 s79, s19, 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 8
+; GFX11-NEXT: s_lshr_b32 s70, s19, 16
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[6:7], 24
; GFX11-NEXT: s_lshr_b64 s[88:89], s[20:21], 24
; GFX11-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 27
-; GFX11-NEXT: s_lshr_b32 s42, s14, 16
+; GFX11-NEXT: v_writelane_b32 v26, s75, 9
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
; GFX11-NEXT: s_lshr_b64 s[92:93], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[94:95], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[0:1], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 28
-; GFX11-NEXT: s_lshr_b32 s42, s14, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 29
-; GFX11-NEXT: s_lshr_b32 s42, s41, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 30
-; GFX11-NEXT: s_lshr_b32 s42, s41, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 31
-; GFX11-NEXT: s_lshr_b32 s42, s41, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s40, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s40, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s29, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s29, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s29, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s28, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s28, 8
+; GFX11-NEXT: s_lshr_b64 vcc, s[0:1], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 6
+; GFX11-NEXT: v_writelane_b32 v26, s75, 7
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 4
+; GFX11-NEXT: v_writelane_b32 v26, s75, 5
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[26:27], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 2
+; GFX11-NEXT: v_writelane_b32 v26, s75, 3
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[24:25], 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 7
-; GFX11-NEXT: s_lshr_b64 s[42:43], s[4:5], 24
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: v_writelane_b32 v26, s74, 0
+; GFX11-NEXT: v_writelane_b32 v26, s75, 1
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[22:23], 24
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, s103, s30, v1
-; GFX11-NEXT: v_readlane_b32 s30, v24, 7
-; GFX11-NEXT: v_readlane_b32 s31, v24, 8
-; GFX11-NEXT: v_readlane_b32 s103, v24, 5
+; GFX11-NEXT: v_readlane_b32 s73, v25, 7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v2, s73, vcc_lo, v1
+; GFX11-NEXT: v_readlane_b32 s73, v25, 8
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_perm_b32 v18, s84, s83, v1
-; GFX11-NEXT: v_perm_b32 v3, s0, s104, v1
-; GFX11-NEXT: v_perm_b32 v4, s1, s102, v1
-; GFX11-NEXT: v_perm_b32 v5, s100, s94, v1
-; GFX11-NEXT: v_perm_b32 v19, s81, s80, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-NEXT: v_perm_b32 v6, s2, s101, v1
-; GFX11-NEXT: v_perm_b32 v7, s3, s99, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v3, s0, s73, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 6
+; GFX11-NEXT: v_perm_b32 v21, s70, s79, v1
+; GFX11-NEXT: v_perm_b32 v22, s77, s78, v1
+; GFX11-NEXT: v_perm_b32 v17, s21, s68, v1
; GFX11-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11-NEXT: v_perm_b32 v8, s97, s92, v1
+; GFX11-NEXT: v_perm_b32 v4, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 2
+; GFX11-NEXT: v_readlane_b32 s1, v25, 5
+; GFX11-NEXT: v_readlane_b32 s70, v23, 20
+; GFX11-NEXT: v_readlane_b32 s68, v23, 18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v5, s0, s94, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 3
+; GFX11-NEXT: v_perm_b32 v6, s2, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v7, s3, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 29
+; GFX11-NEXT: v_perm_b32 v8, s0, s92, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 30
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v9, s16, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 28
+; GFX11-NEXT: v_perm_b32 v10, s17, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 24
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v11, s0, s90, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 25
+; GFX11-NEXT: v_perm_b32 v12, s18, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v13, s19, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 21
+; GFX11-NEXT: v_perm_b32 v14, s0, s88, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, s0, s74, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 22
+; GFX11-NEXT: v_perm_b32 v16, s20, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v18, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 31
+; GFX11-NEXT: v_readlane_b32 s1, v25, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v19, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 26
+; GFX11-NEXT: v_readlane_b32 s1, v26, 27
; GFX11-NEXT: v_or_b32_e32 v3, v4, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v19
-; GFX11-NEXT: v_perm_b32 v11, s86, s90, v1
-; GFX11-NEXT: v_perm_b32 v20, s71, s70, v1
-; GFX11-NEXT: v_perm_b32 v21, s67, s66, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v20, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 0
; GFX11-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v5, v7, v5
-; GFX11-NEXT: v_perm_b32 v9, s16, s98, v1
-; GFX11-NEXT: v_perm_b32 v10, s17, s96, v1
-; GFX11-NEXT: v_perm_b32 v12, s18, s87, v1
-; GFX11-NEXT: v_perm_b32 v13, s19, s85, v1
-; GFX11-NEXT: v_perm_b32 v22, s64, s55, v1
+; GFX11-NEXT: v_perm_b32 v6, s102, s101, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
+; GFX11-NEXT: v_readlane_b32 s1, v26, 1
+; GFX11-NEXT: v_perm_b32 v14, s7, s36, v1
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v21
-; GFX11-NEXT: v_perm_b32 v14, s82, s88, v1
-; GFX11-NEXT: v_perm_b32 v17, s21, s65, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v22
; GFX11-NEXT: v_or_b32_e32 v2, v9, v2
; GFX11-NEXT: v_or_b32_e32 v3, v10, v3
; GFX11-NEXT: v_or_b32_e32 v4, v12, v4
; GFX11-NEXT: v_or_b32_e32 v5, v13, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v22
-; GFX11-NEXT: v_perm_b32 v15, s69, s78, v1
-; GFX11-NEXT: v_perm_b32 v16, s20, s68, v1
-; GFX11-NEXT: v_perm_b32 v6, s53, s52, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:16
-; GFX11-NEXT: v_or_b32_e32 v3, v17, v8
-; GFX11-NEXT: v_perm_b32 v8, s48, s39, v1
-; GFX11-NEXT: v_perm_b32 v9, s22, s54, v1
+; GFX11-NEXT: v_perm_b32 v9, s22, s76, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v15
-; GFX11-NEXT: v_or_b32_e32 v2, v16, v7
-; GFX11-NEXT: v_perm_b32 v5, s50, s72, v1
-; GFX11-NEXT: v_perm_b32 v7, s23, s51, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_perm_b32 v11, s25, s38, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-NEXT: v_readlane_b32 s0, v25, 6
+; GFX11-NEXT: v_perm_b32 v11, s25, s87, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:16
+; GFX11-NEXT: v_or_b32_e32 v2, v16, v7
+; GFX11-NEXT: v_or_b32_e32 v3, v17, v8
+; GFX11-NEXT: v_perm_b32 v5, s99, s0, v1
+; GFX11-NEXT: v_perm_b32 v7, s23, s100, v1
+; GFX11-NEXT: v_perm_b32 v8, s97, s96, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 2
+; GFX11-NEXT: v_readlane_b32 s1, v26, 3
; GFX11-NEXT: v_or_b32_e32 v4, v9, v10
+; GFX11-NEXT: v_perm_b32 v9, s24, s98, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-NEXT: v_perm_b32 v12, s86, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v7, v6
-; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
-; GFX11-NEXT: v_perm_b32 v11, s0, s74, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 3
-; GFX11-NEXT: v_readlane_b32 s1, v25, 4
-; GFX11-NEXT: v_perm_b32 v9, s24, s49, v1
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:32
-; GFX11-NEXT: v_perm_b32 v12, s37, s76, v1
-; GFX11-NEXT: v_perm_b32 v8, s26, s36, v1
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 7
+; GFX11-NEXT: v_readlane_b32 s0, v26, 4
+; GFX11-NEXT: v_readlane_b32 s1, v26, 5
; GFX11-NEXT: v_or_b32_e32 v6, v9, v10
-; GFX11-NEXT: v_perm_b32 v10, s34, vcc_hi, v1
+; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
+; GFX11-NEXT: v_perm_b32 v8, s26, s85, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; GFX11-NEXT: v_perm_b32 v4, s28, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 5
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v10, s83, s82, v1
+; GFX11-NEXT: v_perm_b32 v11, s81, s0, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:32
+; GFX11-NEXT: v_perm_b32 v2, s71, s69, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 6
; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX11-NEXT: v_perm_b32 v9, s27, s35, v1
+; GFX11-NEXT: v_perm_b32 v9, s27, s84, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v4, s28, s80, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s29, s67, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_perm_b32 v10, s29, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 1
-; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_readlane_b32 s1, v26, 31
+; GFX11-NEXT: v_perm_b32 v12, s66, s0, v1
+; GFX11-NEXT: v_readlane_b32 s1, v26, 7
; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s62, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 2
-; GFX11-NEXT: v_readlane_b32 s104, v24, 6
+; GFX11-NEXT: v_perm_b32 v4, s40, s65, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
+; GFX11-NEXT: v_perm_b32 v10, s55, s54, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 8
+; GFX11-NEXT: v_readlane_b32 s1, v26, 9
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:48
-; GFX11-NEXT: v_readlane_b32 s102, v24, 4
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_perm_b32 v5, s41, s64, v1
+; GFX11-NEXT: v_perm_b32 v11, s62, s0, v1
+; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
+; GFX11-NEXT: v_readlane_b32 s0, v26, 10
+; GFX11-NEXT: v_readlane_b32 s1, v26, 11
+; GFX11-NEXT: v_perm_b32 v8, s14, s63, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s15, s52, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-NEXT: v_perm_b32 v12, s61, s0, v1
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: v_readlane_b32 s0, v26, 12
+; GFX11-NEXT: v_readlane_b32 s1, v26, 13
+; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
+; GFX11-NEXT: v_perm_b32 v8, s12, s58, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-NEXT: v_perm_b32 v10, s56, s57, v1
+; GFX11-NEXT: v_perm_b32 v11, s51, s0, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:64
+; GFX11-NEXT: v_perm_b32 v2, s49, s47, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 14
+; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX11-NEXT: v_perm_b32 v9, s13, s59, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v4, s10, s50, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s11, s46, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-NEXT: v_perm_b32 v12, s48, s0, v1
+; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
+; GFX11-NEXT: v_perm_b32 v4, s8, s44, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
-; GFX11-NEXT: v_perm_b32 v4, s40, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 30
+; GFX11-NEXT: v_perm_b32 v10, s39, s43, v1
+; GFX11-NEXT: v_perm_b32 v11, s42, s72, v1
+; GFX11-NEXT: v_readlane_b32 s1, v26, 15
+; GFX11-NEXT: v_perm_b32 v12, s6, s38, v1
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_perm_b32 v5, s9, s45, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-NEXT: v_readlane_b32 s0, v26, 16
+; GFX11-NEXT: v_readlane_b32 s1, v26, 17
+; GFX11-NEXT: v_readlane_b32 s102, v24, 4
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
+; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s101, v24, 3
; GFX11-NEXT: v_readlane_b32 s100, v24, 2
; GFX11-NEXT: v_readlane_b32 s99, v24, 1
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 0
-; GFX11-NEXT: v_readlane_b32 s1, v26, 26
; GFX11-NEXT: v_readlane_b32 s98, v24, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
+; GFX11-NEXT: v_perm_b32 v12, s31, s0, v1
; GFX11-NEXT: v_readlane_b32 s97, v23, 31
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
-; GFX11-NEXT: v_perm_b32 v5, s41, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 28
; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
+; GFX11-NEXT: v_perm_b32 v13, s4, s34, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-NEXT: v_readlane_b32 s87, v23, 29
; GFX11-NEXT: v_readlane_b32 s86, v23, 28
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: v_perm_b32 v11, s0, s60, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 25
; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_readlane_b32 s84, v23, 26
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:64
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
-; GFX11-NEXT: v_perm_b32 v6, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 29
-; GFX11-NEXT: v_readlane_b32 s1, v26, 21
+; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
+; GFX11-NEXT: v_perm_b32 v13, s104, s103, v1
+; GFX11-NEXT: v_perm_b32 v1, s5, s30, v1
+; GFX11-NEXT: v_readlane_b32 s30, v24, 7
+; GFX11-NEXT: v_readlane_b32 s31, v24, 8
+; GFX11-NEXT: v_readlane_b32 s104, v24, 6
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT: v_readlane_b32 s103, v24, 5
; GFX11-NEXT: v_readlane_b32 s83, v23, 25
; GFX11-NEXT: v_readlane_b32 s82, v23, 24
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-NEXT: v_perm_b32 v8, s14, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 27
; GFX11-NEXT: v_readlane_b32 s81, v23, 23
+; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
+; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
; GFX11-NEXT: v_readlane_b32 s80, v23, 22
; GFX11-NEXT: v_readlane_b32 s71, v23, 21
-; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
-; GFX11-NEXT: v_perm_b32 v10, s15, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 23
-; GFX11-NEXT: v_readlane_b32 s70, v23, 20
; GFX11-NEXT: v_readlane_b32 s69, v23, 19
-; GFX11-NEXT: v_readlane_b32 s68, v23, 18
-; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s58, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 24
; GFX11-NEXT: v_readlane_b32 s67, v23, 17
; GFX11-NEXT: v_readlane_b32 s66, v23, 16
; GFX11-NEXT: v_readlane_b32 s65, v23, 15
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-NEXT: v_perm_b32 v8, s12, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 20
; GFX11-NEXT: v_readlane_b32 s64, v23, 14
; GFX11-NEXT: v_readlane_b32 s55, v23, 13
; GFX11-NEXT: v_readlane_b32 s54, v23, 12
-; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 22
-; GFX11-NEXT: v_readlane_b32 s1, v26, 16
; GFX11-NEXT: v_readlane_b32 s53, v23, 11
; GFX11-NEXT: v_readlane_b32 s52, v23, 10
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
-; GFX11-NEXT: v_perm_b32 v9, s13, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 18
; GFX11-NEXT: v_readlane_b32 s51, v23, 9
; GFX11-NEXT: v_readlane_b32 s50, v23, 8
; GFX11-NEXT: v_readlane_b32 s49, v23, 7
-; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
-; GFX11-NEXT: v_perm_b32 v11, s0, s56, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 15
; GFX11-NEXT: v_readlane_b32 s48, v23, 6
; GFX11-NEXT: v_readlane_b32 s39, v23, 5
; GFX11-NEXT: v_readlane_b32 s38, v23, 4
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 19
-; GFX11-NEXT: v_readlane_b32 s1, v26, 11
; GFX11-NEXT: v_readlane_b32 s37, v23, 3
; GFX11-NEXT: v_readlane_b32 s36, v23, 2
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_perm_b32 v4, s10, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 17
; GFX11-NEXT: v_readlane_b32 s35, v23, 1
; GFX11-NEXT: v_readlane_b32 s34, v23, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s11, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 13
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s46, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
-; GFX11-NEXT: v_perm_b32 v4, s8, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 12
-; GFX11-NEXT: v_readlane_b32 s1, v26, 6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-NEXT: v_perm_b32 v5, s9, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
-; GFX11-NEXT: v_perm_b32 v11, s0, s44, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 9
-; GFX11-NEXT: v_readlane_b32 s1, v26, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-NEXT: v_perm_b32 v12, s6, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
-; GFX11-NEXT: v_perm_b32 v14, s7, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
-; GFX11-NEXT: v_perm_b32 v12, s0, s42, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT: v_perm_b32 v13, s4, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
-; GFX11-NEXT: v_perm_b32 v13, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-NEXT: v_perm_b32 v1, s5, s0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
-; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
; GFX11-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-NEXT: s_clause 0x3 ; 16-byte Folded Reload
; GFX11-NEXT: scratch_load_b32 v23, off, s32
@@ -11482,144 +11746,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_mov_b32 exec_lo, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr104
-; GFX11-NEXT: ; implicit-def: $sgpr103
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $sgpr102
-; GFX11-NEXT: ; implicit-def: $sgpr84
-; GFX11-NEXT: ; implicit-def: $sgpr83
-; GFX11-NEXT: ; implicit-def: $sgpr101
-; GFX11-NEXT: ; implicit-def: $sgpr100
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr99
-; GFX11-NEXT: ; implicit-def: $sgpr81
-; GFX11-NEXT: ; implicit-def: $sgpr80
-; GFX11-NEXT: ; implicit-def: $sgpr98
-; GFX11-NEXT: ; implicit-def: $sgpr97
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr96
-; GFX11-NEXT: ; implicit-def: $sgpr71
-; GFX11-NEXT: ; implicit-def: $sgpr70
-; GFX11-NEXT: ; implicit-def: $sgpr87
-; GFX11-NEXT: ; implicit-def: $sgpr86
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr85
-; GFX11-NEXT: ; implicit-def: $sgpr67
-; GFX11-NEXT: ; implicit-def: $sgpr66
-; GFX11-NEXT: ; implicit-def: $sgpr68
-; GFX11-NEXT: ; implicit-def: $sgpr82
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr65
-; GFX11-NEXT: ; implicit-def: $sgpr64
-; GFX11-NEXT: ; implicit-def: $sgpr55
-; GFX11-NEXT: ; implicit-def: $sgpr54
-; GFX11-NEXT: ; implicit-def: $sgpr69
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr51
-; GFX11-NEXT: ; implicit-def: $sgpr53
-; GFX11-NEXT: ; implicit-def: $sgpr52
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr50
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -18135,8 +18261,10 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_cbranch_vccnz .LBB15_5
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
@@ -19133,8 +19261,10 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; VI-NEXT: s_cbranch_vccnz .LBB15_5
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
@@ -20057,8 +20187,10 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB15_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
@@ -20553,7 +20685,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB15_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB15_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -20699,9 +20831,17 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB15_3
-; GFX11-TRUE16-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB15_3
+; GFX11-TRUE16-NEXT: .LBB15_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB15_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
@@ -20992,15 +21132,12 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-TRUE16-NEXT: .LBB15_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB15_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB15_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB15_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v32i32_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -21127,7 +21264,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB15_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB15_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -21273,9 +21410,17 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB15_3
-; GFX11-FAKE16-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB15_3
+; GFX11-FAKE16-NEXT: .LBB15_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB15_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
@@ -21566,15 +21711,12 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-FAKE16-NEXT: .LBB15_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB15_5: ; %end
; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB15_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB15_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -22437,104 +22579,100 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: v_writelane_b32 v33, s30, 34
; SI-NEXT: v_writelane_b32 v33, s31, 35
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_readfirstlane_b32 s70, v17
-; SI-NEXT: v_readfirstlane_b32 s71, v16
-; SI-NEXT: v_readfirstlane_b32 s80, v15
-; SI-NEXT: v_readfirstlane_b32 s81, v14
-; SI-NEXT: v_readfirstlane_b32 s82, v13
-; SI-NEXT: v_readfirstlane_b32 s83, v12
-; SI-NEXT: v_readfirstlane_b32 s84, v11
-; SI-NEXT: v_readfirstlane_b32 s85, v10
-; SI-NEXT: v_readfirstlane_b32 s86, v9
-; SI-NEXT: v_readfirstlane_b32 s87, v8
-; SI-NEXT: v_readfirstlane_b32 s96, v7
-; SI-NEXT: v_readfirstlane_b32 s97, v6
-; SI-NEXT: v_readfirstlane_b32 s98, v5
-; SI-NEXT: v_readfirstlane_b32 s99, v4
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_readfirstlane_b32 s7, v2
-; SI-NEXT: v_readfirstlane_b32 s8, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v17
+; SI-NEXT: v_readfirstlane_b32 s69, v16
+; SI-NEXT: v_readfirstlane_b32 s70, v15
+; SI-NEXT: v_readfirstlane_b32 s71, v14
+; SI-NEXT: v_readfirstlane_b32 s80, v13
+; SI-NEXT: v_readfirstlane_b32 s81, v12
+; SI-NEXT: v_readfirstlane_b32 s82, v11
+; SI-NEXT: v_readfirstlane_b32 s83, v10
+; SI-NEXT: v_readfirstlane_b32 s84, v9
+; SI-NEXT: v_readfirstlane_b32 s85, v8
+; SI-NEXT: v_readfirstlane_b32 s86, v7
+; SI-NEXT: v_readfirstlane_b32 s87, v6
+; SI-NEXT: v_readfirstlane_b32 s96, v5
+; SI-NEXT: v_readfirstlane_b32 s97, v4
+; SI-NEXT: v_readfirstlane_b32 s98, v3
+; SI-NEXT: v_readfirstlane_b32 s99, v2
+; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s9, v0
+; SI-NEXT: v_readfirstlane_b32 s7, v0
; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s17, 0xffff0000
+; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v34, s4, 1
-; SI-NEXT: s_lshl_b32 s4, s17, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 0
-; SI-NEXT: s_and_b32 s4, s16, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 2
; SI-NEXT: s_lshl_b32 s4, s16, 16
-; SI-NEXT: s_and_b32 s11, s70, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s70, 16
-; SI-NEXT: s_and_b32 s13, s71, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s71, 16
-; SI-NEXT: s_and_b32 s15, s80, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s80, 16
-; SI-NEXT: s_and_b32 s41, s81, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s81, 16
-; SI-NEXT: s_and_b32 s43, s82, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s82, 16
-; SI-NEXT: s_and_b32 s45, s83, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s83, 16
-; SI-NEXT: s_and_b32 s47, s84, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s84, 16
-; SI-NEXT: s_and_b32 s57, s85, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s85, 16
-; SI-NEXT: s_and_b32 s59, s86, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s86, 16
-; SI-NEXT: s_and_b32 s61, s87, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s87, 16
-; SI-NEXT: s_and_b32 s63, s96, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s96, 16
-; SI-NEXT: s_and_b32 s73, s97, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s97, 16
-; SI-NEXT: s_and_b32 s75, s98, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s98, 16
-; SI-NEXT: s_and_b32 s77, s99, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s99, 16
-; SI-NEXT: s_and_b32 s79, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s6, 16
-; SI-NEXT: s_and_b32 s89, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s7, 16
-; SI-NEXT: s_and_b32 s91, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s8, 16
-; SI-NEXT: s_and_b32 s93, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s9, 16
-; SI-NEXT: s_and_b32 s95, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s29, 16
-; SI-NEXT: s_and_b32 s31, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s28, 16
-; SI-NEXT: s_and_b32 s35, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s27, 16
-; SI-NEXT: s_and_b32 s37, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s26, 16
-; SI-NEXT: s_and_b32 s39, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s25, 16
-; SI-NEXT: s_and_b32 s49, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s24, 16
-; SI-NEXT: s_and_b32 s51, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s23, 16
-; SI-NEXT: s_and_b32 s53, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s22, 16
-; SI-NEXT: s_and_b32 s55, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s21, 16
-; SI-NEXT: s_and_b32 s65, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s20, 16
-; SI-NEXT: s_and_b32 s67, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s19, 16
-; SI-NEXT: s_and_b32 s69, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s18, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 3
+; SI-NEXT: s_and_b32 s9, s68, 0xffff0000
+; SI-NEXT: s_lshl_b32 s8, s68, 16
+; SI-NEXT: s_and_b32 s11, s69, 0xffff0000
+; SI-NEXT: s_lshl_b32 s10, s69, 16
+; SI-NEXT: s_and_b32 s13, s70, 0xffff0000
+; SI-NEXT: s_lshl_b32 s12, s70, 16
+; SI-NEXT: s_and_b32 s15, s71, 0xffff0000
+; SI-NEXT: s_lshl_b32 s14, s71, 16
+; SI-NEXT: s_and_b32 s41, s80, 0xffff0000
+; SI-NEXT: s_lshl_b32 s40, s80, 16
+; SI-NEXT: s_and_b32 s43, s81, 0xffff0000
+; SI-NEXT: s_lshl_b32 s42, s81, 16
+; SI-NEXT: s_and_b32 s45, s82, 0xffff0000
+; SI-NEXT: s_lshl_b32 s44, s82, 16
+; SI-NEXT: s_and_b32 s47, s83, 0xffff0000
+; SI-NEXT: s_lshl_b32 s46, s83, 16
+; SI-NEXT: s_and_b32 s57, s84, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s84, 16
+; SI-NEXT: s_and_b32 s59, s85, 0xffff0000
+; SI-NEXT: s_lshl_b32 s58, s85, 16
+; SI-NEXT: s_and_b32 s61, s86, 0xffff0000
+; SI-NEXT: s_lshl_b32 s60, s86, 16
+; SI-NEXT: s_and_b32 s63, s87, 0xffff0000
+; SI-NEXT: s_lshl_b32 s62, s87, 16
+; SI-NEXT: s_and_b32 s73, s96, 0xffff0000
+; SI-NEXT: s_lshl_b32 s72, s96, 16
+; SI-NEXT: s_and_b32 s75, s97, 0xffff0000
+; SI-NEXT: s_lshl_b32 s74, s97, 16
+; SI-NEXT: s_and_b32 s77, s98, 0xffff0000
+; SI-NEXT: s_lshl_b32 s76, s98, 16
+; SI-NEXT: s_and_b32 s79, s99, 0xffff0000
+; SI-NEXT: s_lshl_b32 s78, s99, 16
+; SI-NEXT: s_and_b32 s89, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s6, 16
+; SI-NEXT: s_and_b32 s91, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s90, s7, 16
+; SI-NEXT: s_and_b32 s93, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s92, s29, 16
+; SI-NEXT: s_and_b32 s95, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 s94, s28, 16
+; SI-NEXT: s_and_b32 vcc_hi, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, s27, 16
+; SI-NEXT: s_and_b32 s31, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 s30, s26, 16
+; SI-NEXT: s_and_b32 s35, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s25, 16
+; SI-NEXT: s_and_b32 s37, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s36, s24, 16
+; SI-NEXT: s_and_b32 s39, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s38, s23, 16
+; SI-NEXT: s_and_b32 s49, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s48, s22, 16
+; SI-NEXT: s_and_b32 s51, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s50, s21, 16
+; SI-NEXT: s_and_b32 s53, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s20, 16
+; SI-NEXT: s_and_b32 s55, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s19, 16
+; SI-NEXT: s_and_b32 s65, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s64, s18, 16
+; SI-NEXT: s_and_b32 s67, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s66, s17, 16
+; SI-NEXT: v_writelane_b32 v34, s4, 0
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB17_3
; SI-NEXT: .LBB17_2:
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr67
; SI-NEXT: ; implicit-def: $sgpr64
@@ -22555,6 +22693,8 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: ; implicit-def: $sgpr35
; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr92
@@ -22593,17 +22733,19 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: ; implicit-def: $sgpr13
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: .LBB17_3: ; %Flow
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_mov_b32 s4, s10
-; SI-NEXT: s_mov_b32 s5, s11
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_mov_b32 s4, s8
+; SI-NEXT: s_mov_b32 s5, s9
+; SI-NEXT: s_mov_b32 s8, s10
+; SI-NEXT: s_mov_b32 s9, s11
; SI-NEXT: s_mov_b32 s10, s12
; SI-NEXT: s_mov_b32 s11, s13
; SI-NEXT: s_mov_b32 s12, s14
@@ -22630,14 +22772,11 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: s_mov_b32 s73, s75
; SI-NEXT: s_mov_b32 s74, s76
; SI-NEXT: s_mov_b32 s75, s77
-; SI-NEXT: s_mov_b32 s76, s78
-; SI-NEXT: s_mov_b32 s77, s79
-; SI-NEXT: v_readlane_b32 s78, v34, 0
-; SI-NEXT: v_readlane_b32 s79, v34, 1
-; SI-NEXT: s_cbranch_vccnz .LBB17_5
+; SI-NEXT: v_readlane_b32 s76, v34, 0
+; SI-NEXT: v_readlane_b32 s77, v34, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -22651,9 +22790,8 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: s_add_i32 s8, s8, 3
; SI-NEXT: s_add_i32 s7, s7, 3
+; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s99, s99, 3
; SI-NEXT: s_add_i32 s98, s98, 3
; SI-NEXT: s_add_i32 s97, s97, 3
@@ -22668,199 +22806,196 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; SI-NEXT: s_add_i32 s80, s80, 3
; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s70, s70, 3
-; SI-NEXT: s_and_b32 s77, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s6, 16
-; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
-; SI-NEXT: s_and_b32 s5, s70, 0xffff0000
-; SI-NEXT: s_lshl_b32 s4, s70, 16
-; SI-NEXT: s_and_b32 s11, s71, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s71, 16
-; SI-NEXT: s_and_b32 s13, s80, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s80, 16
-; SI-NEXT: s_and_b32 s15, s81, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s81, 16
-; SI-NEXT: s_and_b32 s41, s82, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s82, 16
-; SI-NEXT: s_and_b32 s43, s83, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s83, 16
-; SI-NEXT: s_and_b32 s45, s84, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s84, 16
-; SI-NEXT: s_and_b32 s47, s85, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s85, 16
-; SI-NEXT: s_and_b32 s57, s86, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s86, 16
-; SI-NEXT: s_and_b32 s59, s87, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s87, 16
-; SI-NEXT: s_and_b32 s61, s96, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s96, 16
-; SI-NEXT: s_and_b32 s63, s97, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s97, 16
-; SI-NEXT: s_and_b32 s73, s98, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s98, 16
-; SI-NEXT: s_and_b32 s75, s99, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s99, 16
-; SI-NEXT: s_and_b32 s89, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s7, 16
-; SI-NEXT: s_and_b32 s91, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s8, 16
-; SI-NEXT: s_and_b32 s93, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s9, 16
-; SI-NEXT: s_and_b32 s95, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s29, 16
-; SI-NEXT: s_and_b32 s31, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s28, 16
-; SI-NEXT: s_and_b32 s35, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s27, 16
-; SI-NEXT: s_and_b32 s37, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s26, 16
-; SI-NEXT: s_and_b32 s39, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s25, 16
-; SI-NEXT: s_and_b32 s49, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s24, 16
-; SI-NEXT: s_and_b32 s51, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s23, 16
-; SI-NEXT: s_and_b32 s53, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s22, 16
-; SI-NEXT: s_and_b32 s55, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s21, 16
-; SI-NEXT: s_and_b32 s65, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s20, 16
-; SI-NEXT: s_and_b32 s67, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s19, 16
-; SI-NEXT: s_and_b32 s69, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s18, 16
-; SI-NEXT: s_and_b32 s79, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s17, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v34, s6, 2
-; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 3
+; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_and_b32 s5, s68, 0xffff0000
+; SI-NEXT: s_lshl_b32 s4, s68, 16
+; SI-NEXT: s_and_b32 s9, s69, 0xffff0000
+; SI-NEXT: s_lshl_b32 s8, s69, 16
+; SI-NEXT: s_and_b32 s11, s70, 0xffff0000
+; SI-NEXT: s_lshl_b32 s10, s70, 16
+; SI-NEXT: s_and_b32 s13, s71, 0xffff0000
+; SI-NEXT: s_lshl_b32 s12, s71, 16
+; SI-NEXT: s_and_b32 s15, s80, 0xffff0000
+; SI-NEXT: s_lshl_b32 s14, s80, 16
+; SI-NEXT: s_and_b32 s41, s81, 0xffff0000
+; SI-NEXT: s_lshl_b32 s40, s81, 16
+; SI-NEXT: s_and_b32 s43, s82, 0xffff0000
+; SI-NEXT: s_lshl_b32 s42, s82, 16
+; SI-NEXT: s_and_b32 s45, s83, 0xffff0000
+; SI-NEXT: s_lshl_b32 s44, s83, 16
+; SI-NEXT: s_and_b32 s47, s84, 0xffff0000
+; SI-NEXT: s_lshl_b32 s46, s84, 16
+; SI-NEXT: s_and_b32 s57, s85, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s85, 16
+; SI-NEXT: s_and_b32 s59, s86, 0xffff0000
+; SI-NEXT: s_lshl_b32 s58, s86, 16
+; SI-NEXT: s_and_b32 s61, s87, 0xffff0000
+; SI-NEXT: s_lshl_b32 s60, s87, 16
+; SI-NEXT: s_and_b32 s63, s96, 0xffff0000
+; SI-NEXT: s_lshl_b32 s62, s96, 16
+; SI-NEXT: s_and_b32 s73, s97, 0xffff0000
+; SI-NEXT: s_lshl_b32 s72, s97, 16
+; SI-NEXT: s_and_b32 s75, s98, 0xffff0000
+; SI-NEXT: s_lshl_b32 s74, s98, 16
+; SI-NEXT: s_and_b32 s79, s99, 0xffff0000
+; SI-NEXT: s_lshl_b32 s78, s99, 16
+; SI-NEXT: s_and_b32 s89, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s6, 16
+; SI-NEXT: s_and_b32 s91, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s90, s7, 16
+; SI-NEXT: s_and_b32 s93, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s92, s29, 16
+; SI-NEXT: s_and_b32 s95, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 s94, s28, 16
+; SI-NEXT: s_and_b32 vcc_hi, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, s27, 16
+; SI-NEXT: s_and_b32 s31, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 s30, s26, 16
+; SI-NEXT: s_and_b32 s35, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s25, 16
+; SI-NEXT: s_and_b32 s37, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s36, s24, 16
+; SI-NEXT: s_and_b32 s39, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s38, s23, 16
+; SI-NEXT: s_and_b32 s49, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s48, s22, 16
+; SI-NEXT: s_and_b32 s51, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s50, s21, 16
+; SI-NEXT: s_and_b32 s53, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s20, 16
+; SI-NEXT: s_and_b32 s55, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s19, 16
+; SI-NEXT: s_and_b32 s65, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s64, s18, 16
+; SI-NEXT: s_and_b32 s67, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s66, s17, 16
+; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s76, s16, 16
; SI-NEXT: .LBB17_5: ; %end
-; SI-NEXT: v_readlane_b32 s6, v34, 2
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_readlane_b32 s6, v34, 3
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s79
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s67
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s66
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s69
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s65
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s68
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s64
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s67
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s55
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s66
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s54
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s65
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s53
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s64
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s52
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s55
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s51
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s54
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s50
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s53
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s49
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s52
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s48
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s51
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s39
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s50
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s38
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s49
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s37
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s48
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s36
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s39
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s35
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s38
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s34
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s31
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s36
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s30
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, vcc_hi
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s34
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, vcc_lo
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s31
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s95
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s94
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s93
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s92
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s93
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s91
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s92
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s90
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s89
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s88
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s89
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s79
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s88
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s78
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s77
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s75
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s76
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s74
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s75
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s73
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s72
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s73
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s63
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s62
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s63
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s61
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s60
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s61
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s59
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s58
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s59
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s57
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s56
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s57
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s47
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s56
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s47
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s45
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s46
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s44
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s43
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s44
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s42
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s41
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s42
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s40
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s15
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s14
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s15
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s13
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s11
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s12
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s10
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mul_f32_e64 v31, 1.0, s5
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
@@ -22933,10 +23068,18 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s46, s46, 3
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -22969,7 +23112,7 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; VI-NEXT: s_add_i32 s43, s43, 3
; VI-NEXT: s_add_i32 s44, s44, 3
; VI-NEXT: s_add_i32 s45, s45, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -23003,8 +23146,6 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v32i32_to_v64bf16_scalar:
; GFX9: ; %bb.0:
@@ -23029,10 +23170,18 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s46, s46, 3
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -23065,7 +23214,7 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX9-NEXT: s_add_i32 s43, s43, 3
; GFX9-NEXT: s_add_i32 s44, s44, 3
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -23099,8 +23248,6 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v32i32_to_v64bf16_scalar:
; GFX11: ; %bb.0:
@@ -23122,11 +23269,16 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -23159,7 +23311,7 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX11-NEXT: s_add_i32 s41, s41, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -23177,8 +23329,6 @@ define inreg <64 x bfloat> @bitcast_v32i32_to_v64bf16_scalar(<32 x i32> inreg %a
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -26695,7 +26845,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_mov_b32_e32 v33, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
@@ -26712,7 +26864,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v40, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_cbranch_vccnz .LBB19_5
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -27082,10 +27234,18 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s50, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -27678,10 +27838,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -27714,7 +27872,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
+; VI-NEXT: .LBB19_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v35, 15
; VI-NEXT: v_readlane_b32 s66, v35, 14
; VI-NEXT: v_readlane_b32 s65, v35, 13
@@ -27793,10 +27951,18 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -28407,10 +28573,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -28443,7 +28607,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v36, 15
; GFX9-NEXT: v_readlane_b32 s66, v36, 14
; GFX9-NEXT: v_readlane_b32 s65, v36, 13
@@ -28523,11 +28687,16 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s37, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-TRUE16-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB19_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s51, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s51, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -29150,8 +29319,6 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v34
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v33.l
; GFX11-TRUE16-NEXT: s_branch .LBB19_5
-; GFX11-TRUE16-NEXT: .LBB19_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB19_2
; GFX11-TRUE16-NEXT: .LBB19_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -29249,11 +29416,16 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s37, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-FAKE16-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB19_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s51, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s51, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -29907,8 +30079,6 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v32, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v36, 16, v33
; GFX11-FAKE16-NEXT: s_branch .LBB19_5
-; GFX11-FAKE16-NEXT: .LBB19_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB19_2
; GFX11-FAKE16-NEXT: .LBB19_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -30478,7 +30648,7 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 16
; SI-NEXT: s_lshr_b32 s39, s7, 16
@@ -30509,11 +30679,51 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -30575,23 +30785,23 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: .LBB21_3: ; %end
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: s_lshl_b32 s47, s34, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: s_lshl_b32 s47, s69, 16
; SI-NEXT: s_or_b32 s17, s17, s47
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s47
; SI-NEXT: s_and_b32 s19, s19, 0xffff
; SI-NEXT: s_lshl_b32 s47, s68, 16
; SI-NEXT: s_or_b32 s19, s19, s47
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s47
; SI-NEXT: s_and_b32 s21, s21, 0xffff
@@ -30734,40 +30944,6 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v32i32_to_v64f16_scalar:
; VI: ; %bb.0:
@@ -30792,10 +30968,18 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s46, s46, 3
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -30828,7 +31012,7 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s43, s43, 3
; VI-NEXT: s_add_i32 s44, s44, 3
; VI-NEXT: s_add_i32 s45, s45, 3
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -30862,8 +31046,6 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v32i32_to_v64f16_scalar:
; GFX9: ; %bb.0:
@@ -30888,10 +31070,18 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s46, s46, 3
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -30924,7 +31114,7 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s43, s43, 3
; GFX9-NEXT: s_add_i32 s44, s44, 3
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -30958,8 +31148,6 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_v32i32_to_v64f16_scalar:
; GFX11: ; %bb.0:
@@ -30981,11 +31169,16 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -31018,7 +31211,7 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s41, s41, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -31036,8 +31229,6 @@ define inreg <64 x half> @bitcast_v32i32_to_v64f16_scalar(<32 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -32060,10 +32251,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v34, s30, 34
; SI-NEXT: v_writelane_b32 v34, s31, 35
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 vcc_hi, s8, 16
; SI-NEXT: v_readfirstlane_b32 s10, v15
; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: v_readfirstlane_b32 s14, v13
@@ -32075,29 +32263,29 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v7
; SI-NEXT: v_readfirstlane_b32 s92, v6
; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s34, v3
-; SI-NEXT: v_readfirstlane_b32 s68, v2
-; SI-NEXT: v_readfirstlane_b32 s80, v1
-; SI-NEXT: v_readfirstlane_b32 s83, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v35, vcc_lo, 0
-; SI-NEXT: s_lshr_b32 s69, s29, 16
-; SI-NEXT: s_lshr_b32 s71, s28, 16
-; SI-NEXT: s_lshr_b32 s82, s27, 16
-; SI-NEXT: s_lshr_b32 s85, s26, 16
-; SI-NEXT: s_lshr_b32 s86, s25, 16
-; SI-NEXT: s_lshr_b32 s87, s24, 16
-; SI-NEXT: s_lshr_b32 s96, s23, 16
-; SI-NEXT: s_lshr_b32 s97, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s7, s19, 16
-; SI-NEXT: s_lshr_b32 s9, s18, 16
-; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: s_lshr_b32 s13, s16, 16
-; SI-NEXT: s_lshr_b32 s65, s10, 16
-; SI-NEXT: s_lshr_b32 s64, s12, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s30, v3
+; SI-NEXT: v_readfirstlane_b32 s34, v2
+; SI-NEXT: v_readfirstlane_b32 s70, v1
+; SI-NEXT: v_readfirstlane_b32 s81, v0
+; SI-NEXT: s_lshr_b32 s35, s29, 16
+; SI-NEXT: s_lshr_b32 s69, s28, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 16
+; SI-NEXT: s_lshr_b32 s83, s26, 16
+; SI-NEXT: s_lshr_b32 s84, s25, 16
+; SI-NEXT: s_lshr_b32 s85, s24, 16
+; SI-NEXT: s_lshr_b32 s86, s23, 16
+; SI-NEXT: s_lshr_b32 s87, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s7, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s6, 16
+; SI-NEXT: s_lshr_b32 s66, s8, 16
+; SI-NEXT: s_lshr_b32 s11, s10, 16
+; SI-NEXT: s_lshr_b32 s13, s12, 16
; SI-NEXT: s_lshr_b32 s15, s14, 16
; SI-NEXT: s_lshr_b32 s73, s72, 16
; SI-NEXT: s_lshr_b32 s75, s74, 16
@@ -32107,74 +32295,75 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s31, s30, 16
-; SI-NEXT: s_lshr_b32 s35, s34, 16
-; SI-NEXT: s_lshr_b32 s70, s68, 16
-; SI-NEXT: s_lshr_b32 s81, s80, 16
-; SI-NEXT: s_lshr_b32 s84, s83, 16
+; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 s71, s70, 16
+; SI-NEXT: s_lshr_b32 s82, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v35, vcc_hi, 1
+; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v35, s65, 2
-; SI-NEXT: v_writelane_b32 v35, s64, 3
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v35, s67, 0
+; SI-NEXT: v_writelane_b32 v35, s66, 1
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s11, 16
+; SI-NEXT: s_lshl_b32 s41, s7, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
-; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s30, 0xffff
; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -32204,22 +32393,31 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s15, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s7
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -32231,7 +32429,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s9
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s99
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -32239,11 +32437,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s98
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s99
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s97
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -32254,7 +32452,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s96
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -32263,11 +32461,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s86
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -32278,7 +32476,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s85
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -32287,11 +32485,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s84
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s83
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -32302,7 +32500,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s80
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -32311,11 +32509,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: v_cvt_f32_f16_e32 v13, s28
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s35
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
@@ -32326,8 +32524,8 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v15, s29
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s84
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s81
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -32335,11 +32533,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s71
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s80
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s68
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
@@ -32347,11 +32545,11 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s34
; SI-NEXT: v_or_b32_e32 v15, v16, v15
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
@@ -32359,10 +32557,10 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
; SI-NEXT: v_or_b32_e32 v16, v18, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v18, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v19, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v19, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v20, s95
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
@@ -32433,10 +32631,9 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v25, v27, v25
; SI-NEXT: v_cvt_f32_f16_e32 v27, s15
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_readlane_b32 s4, v35, 3
; SI-NEXT: v_or_b32_e32 v26, v28, v26
; SI-NEXT: v_cvt_f32_f16_e32 v28, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s13
; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v28
@@ -32444,11 +32641,10 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_readlane_b32 s4, v35, 2
; SI-NEXT: v_cvt_f32_f16_e32 v30, s12
; SI-NEXT: v_or_b32_e32 v27, v28, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s11
; SI-NEXT: v_cvt_f32_f16_e32 v31, s10
; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
@@ -32477,11 +32673,8 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v30, v31, v30
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v33, v31
-; SI-NEXT: s_branch .LBB23_5
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -32514,7 +32707,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: .LBB23_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v34, 34
; SI-NEXT: v_readlane_b32 s31, v34, 35
; SI-NEXT: v_readlane_b32 s99, v34, 33
@@ -32614,10 +32807,18 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s51, 16
; VI-NEXT: v_mov_b32_e32 v16, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -32779,10 +32980,8 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s52, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB23_5
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: s_branch .LBB23_6
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -32815,7 +33014,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB23_5: ; %end
+; VI-NEXT: .LBB23_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v33, 15
; VI-NEXT: v_readlane_b32 s66, v33, 14
; VI-NEXT: v_readlane_b32 s65, v33, 13
@@ -32894,10 +33093,18 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s51, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s50, v16 op_sel_hi:[1,0]
@@ -32931,10 +33138,8 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v18, s54, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB23_5
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: s_branch .LBB23_6
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -32967,7 +33172,7 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB23_5: ; %end
+; GFX9-NEXT: .LBB23_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -33047,11 +33252,16 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s51 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s50 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s49 op_sel_hi:[0,1]
@@ -33085,8 +33295,6 @@ define inreg <32 x i32> @bitcast_v64f16_to_v32i32_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s53 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s52 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB23_5
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -33656,7 +33864,7 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_4
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 16
; SI-NEXT: s_lshr_b32 s39, s7, 16
@@ -33687,11 +33895,51 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB25_3
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -33753,23 +34001,23 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: .LBB25_3: ; %end
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: s_lshl_b32 s47, s34, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: s_lshl_b32 s47, s69, 16
; SI-NEXT: s_or_b32 s17, s17, s47
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s47
; SI-NEXT: s_and_b32 s19, s19, 0xffff
; SI-NEXT: s_lshl_b32 s47, s68, 16
; SI-NEXT: s_or_b32 s19, s19, s47
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s47
; SI-NEXT: s_and_b32 s21, s21, 0xffff
@@ -33912,40 +34160,6 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB25_2
;
; VI-LABEL: bitcast_v32i32_to_v64i16_scalar:
; VI: ; %bb.0:
@@ -33970,10 +34184,18 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_4
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_3
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s46, s46, 3
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -34006,7 +34228,7 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s43, s43, 3
; VI-NEXT: s_add_i32 s44, s44, 3
; VI-NEXT: s_add_i32 s45, s45, 3
-; VI-NEXT: .LBB25_3: ; %end
+; VI-NEXT: .LBB25_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -34040,8 +34262,6 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_4:
-; VI-NEXT: s_branch .LBB25_2
;
; GFX9-LABEL: bitcast_v32i32_to_v64i16_scalar:
; GFX9: ; %bb.0:
@@ -34066,10 +34286,18 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_3
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s46, s46, 3
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -34102,7 +34330,7 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s43, s43, 3
; GFX9-NEXT: s_add_i32 s44, s44, 3
; GFX9-NEXT: s_add_i32 s45, s45, 3
-; GFX9-NEXT: .LBB25_3: ; %end
+; GFX9-NEXT: .LBB25_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -34136,8 +34364,6 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_4:
-; GFX9-NEXT: s_branch .LBB25_2
;
; GFX11-LABEL: bitcast_v32i32_to_v64i16_scalar:
; GFX11: ; %bb.0:
@@ -34159,11 +34385,16 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_3
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -34196,7 +34427,7 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s41, s41, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
-; GFX11-NEXT: .LBB25_3: ; %end
+; GFX11-NEXT: .LBB25_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -34214,8 +34445,6 @@ define inreg <64 x i16> @bitcast_v32i32_to_v64i16_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_4:
-; GFX11-NEXT: s_branch .LBB25_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -35067,13 +35296,10 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v32, s99, 33
; SI-NEXT: v_writelane_b32 v32, s30, 34
; SI-NEXT: v_writelane_b32 v32, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s9, v16
-; SI-NEXT: s_lshr_b32 s14, s9, 16
-; SI-NEXT: v_readfirstlane_b32 s13, v14
-; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
; SI-NEXT: v_readfirstlane_b32 s11, v15
-; SI-NEXT: s_lshr_b32 s72, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s13, v14
; SI-NEXT: v_readfirstlane_b32 s15, v13
; SI-NEXT: v_readfirstlane_b32 s73, v12
; SI-NEXT: v_readfirstlane_b32 s75, v11
@@ -35082,30 +35308,30 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s89, v8
; SI-NEXT: v_readfirstlane_b32 s91, v7
; SI-NEXT: v_readfirstlane_b32 s94, v6
-; SI-NEXT: v_readfirstlane_b32 s31, v5
-; SI-NEXT: v_readfirstlane_b32 s68, v4
-; SI-NEXT: v_readfirstlane_b32 s80, v3
-; SI-NEXT: v_readfirstlane_b32 s83, v2
-; SI-NEXT: v_readfirstlane_b32 s86, v1
-; SI-NEXT: v_readfirstlane_b32 s97, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; SI-NEXT: v_readfirstlane_b32 s34, v4
+; SI-NEXT: v_readfirstlane_b32 s70, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s84, v1
+; SI-NEXT: v_readfirstlane_b32 s87, v0
; SI-NEXT: s_lshr_b32 s92, s29, 16
; SI-NEXT: s_lshr_b32 s95, s28, 16
-; SI-NEXT: s_lshr_b32 s34, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s71, s25, 16
-; SI-NEXT: s_lshr_b32 s81, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s87, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s6, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s10, s17, 16
-; SI-NEXT: s_lshr_b32 s12, s16, 16
-; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s11, 16
+; SI-NEXT: s_lshr_b32 s30, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s69, s25, 16
+; SI-NEXT: s_lshr_b32 s71, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s85, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s10, s9, 16
+; SI-NEXT: s_lshr_b32 s12, s11, 16
+; SI-NEXT: s_lshr_b32 s14, s13, 16
; SI-NEXT: s_lshr_b32 s63, s15, 16
; SI-NEXT: s_lshr_b32 s62, s73, 16
; SI-NEXT: s_lshr_b32 s61, s75, 16
@@ -35114,54 +35340,55 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s68, 16
-; SI-NEXT: s_lshr_b32 s70, s80, 16
-; SI-NEXT: s_lshr_b32 s82, s83, 16
-; SI-NEXT: s_lshr_b32 s85, s86, 16
-; SI-NEXT: s_lshr_b32 s96, s97, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s70, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v33, s72, 1
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v33, s63, 2
-; SI-NEXT: v_writelane_b32 v33, s62, 3
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_writelane_b32 v33, s63, 1
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s10, 16
+; SI-NEXT: s_lshl_b32 s41, s6, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -35169,23 +35396,23 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s54, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s55, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -35201,92 +35428,109 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s59, s4, s5
; SI-NEXT: s_and_b32 s4, s77, 0xffff
; SI-NEXT: s_lshl_b32 s5, s60, 16
-; SI-NEXT: s_mov_b32 s76, s60
+; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: s_or_b32 s60, s4, s5
; SI-NEXT: s_and_b32 s4, s75, 0xffff
; SI-NEXT: s_lshl_b32 s5, s61, 16
-; SI-NEXT: s_mov_b32 s74, s61
+; SI-NEXT: s_mov_b32 s76, s61
; SI-NEXT: s_or_b32 s61, s4, s5
; SI-NEXT: s_and_b32 s4, s73, 0xffff
; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_mov_b32 s72, s62
; SI-NEXT: s_or_b32 s62, s4, s5
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b32 s74, s60
+; SI-NEXT: s_mov_b32 s76, s61
+; SI-NEXT: s_mov_b32 s72, s62
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_add_i32 s37, s5, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -35298,35 +35542,35 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s97, s97, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
+; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s34, s34, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s54, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s55, s4, 0x30000
@@ -35351,27 +35595,26 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s74, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s75, s75, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s73, s73, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s72, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s15, s15, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 2
+; SI-NEXT: v_readlane_b32 s5, v33, 1
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 1
+; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s63, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s13, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
@@ -35379,21 +35622,20 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_add_i32 s64, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s65, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s66, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s67, s4, 0x30000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_readlane_b32 s30, v32, 34
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
@@ -35468,13 +35710,6 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: s_mov_b32 s76, s60
-; SI-NEXT: s_mov_b32 s74, s61
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v64i16_to_v32i32_scalar:
; VI: ; %bb.0:
@@ -35499,10 +35734,18 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s46, 3
; VI-NEXT: s_and_b32 s4, s46, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -35663,7 +35906,7 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s45, s4, 0x30000
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -35697,8 +35940,6 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v64i16_to_v32i32_scalar:
; GFX9: ; %bb.0:
@@ -35756,10 +35997,18 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -35792,10 +36041,8 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -35828,7 +36075,7 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -35908,11 +36155,16 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -35946,8 +36198,6 @@ define inreg <32 x i32> @bitcast_v64i16_to_v32i32_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -36265,10 +36515,18 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v31, s67, 1.0
; SI-NEXT: v_add_f32_e64 v30, s66, 1.0
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -36301,10 +36559,8 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -36337,7 +36593,7 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -36416,10 +36672,18 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v31, s67, 1.0
; VI-NEXT: v_add_f32_e64 v30, s66, 1.0
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -36452,10 +36716,8 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -36488,7 +36750,7 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -36567,10 +36829,18 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX9-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -36603,10 +36873,8 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -36639,7 +36907,7 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -36719,11 +36987,16 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX11-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -36757,8 +37030,6 @@ define inreg <16 x i64> @bitcast_v32f32_to_v16i64_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB29_5
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -37066,10 +37337,18 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -37102,7 +37381,7 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -37136,8 +37415,6 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v16i64_to_v32f32_scalar:
; VI: ; %bb.0:
@@ -37162,10 +37439,18 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -37198,7 +37483,7 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -37232,8 +37517,6 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v16i64_to_v32f32_scalar:
; GFX9: ; %bb.0:
@@ -37258,10 +37541,18 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_3
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -37294,7 +37585,7 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB31_3: ; %end
+; GFX9-NEXT: .LBB31_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -37328,8 +37619,6 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_4:
-; GFX9-NEXT: s_branch .LBB31_2
;
; GFX11-LABEL: bitcast_v16i64_to_v32f32_scalar:
; GFX11: ; %bb.0:
@@ -37351,11 +37640,16 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_3
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -37388,7 +37682,7 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB31_3: ; %end
+; GFX11-NEXT: .LBB31_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -37407,8 +37701,6 @@ define inreg <32 x float> @bitcast_v16i64_to_v32f32_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_4:
-; GFX11-NEXT: s_branch .LBB31_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -37687,10 +37979,18 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v31, s67, 1.0
; SI-NEXT: v_add_f32_e64 v30, s66, 1.0
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -37723,10 +38023,8 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -37759,7 +38057,7 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -37838,10 +38136,18 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v31, s67, 1.0
; VI-NEXT: v_add_f32_e64 v30, s66, 1.0
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -37874,10 +38180,8 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -37910,7 +38214,7 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -37989,10 +38293,18 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX9-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -38025,10 +38337,8 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -38061,7 +38371,7 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -38141,11 +38451,16 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v31, s67, 1.0
; GFX11-NEXT: v_add_f32_e64 v30, s66, 1.0
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
@@ -38179,8 +38494,6 @@ define inreg <16 x double> @bitcast_v32f32_to_v16f64_scalar(<32 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB33_5
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -38450,10 +38763,18 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -38470,10 +38791,8 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -38506,7 +38825,7 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: .LBB35_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -38585,10 +38904,18 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -38605,10 +38932,8 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -38641,7 +38966,7 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
+; VI-NEXT: .LBB35_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -38720,10 +39045,18 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -38740,10 +39073,8 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -38776,7 +39107,7 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -38856,11 +39187,16 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
@@ -38878,8 +39214,6 @@ define inreg <32 x float> @bitcast_v16f64_to_v32f32_scalar(<16 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB35_5
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -42919,72 +43253,72 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v63, s30, 34
; SI-NEXT: v_writelane_b32 v63, s31, 35
; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: v_readfirstlane_b32 s45, v18
-; SI-NEXT: v_readfirstlane_b32 s44, v17
-; SI-NEXT: v_readfirstlane_b32 s47, v16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
-; SI-NEXT: v_readfirstlane_b32 s57, v14
-; SI-NEXT: v_readfirstlane_b32 s56, v13
-; SI-NEXT: v_readfirstlane_b32 s59, v12
-; SI-NEXT: v_readfirstlane_b32 s58, v11
-; SI-NEXT: v_readfirstlane_b32 s61, v10
-; SI-NEXT: v_readfirstlane_b32 s60, v9
-; SI-NEXT: v_readfirstlane_b32 s63, v8
-; SI-NEXT: v_readfirstlane_b32 s62, v7
-; SI-NEXT: v_readfirstlane_b32 s73, v6
-; SI-NEXT: v_readfirstlane_b32 s72, v5
-; SI-NEXT: v_readfirstlane_b32 s75, v4
-; SI-NEXT: v_readfirstlane_b32 s74, v3
-; SI-NEXT: v_readfirstlane_b32 s77, v2
+; SI-NEXT: v_readfirstlane_b32 s43, v18
+; SI-NEXT: v_readfirstlane_b32 s42, v17
+; SI-NEXT: v_readfirstlane_b32 s45, v16
+; SI-NEXT: v_readfirstlane_b32 s44, v15
+; SI-NEXT: v_readfirstlane_b32 s47, v14
+; SI-NEXT: v_readfirstlane_b32 s46, v13
+; SI-NEXT: v_readfirstlane_b32 s57, v12
+; SI-NEXT: v_readfirstlane_b32 s56, v11
+; SI-NEXT: v_readfirstlane_b32 s59, v10
+; SI-NEXT: v_readfirstlane_b32 s58, v9
+; SI-NEXT: v_readfirstlane_b32 s61, v8
+; SI-NEXT: v_readfirstlane_b32 s60, v7
+; SI-NEXT: v_readfirstlane_b32 s63, v6
+; SI-NEXT: v_readfirstlane_b32 s62, v5
+; SI-NEXT: v_readfirstlane_b32 s73, v4
+; SI-NEXT: v_readfirstlane_b32 s72, v3
+; SI-NEXT: v_readfirstlane_b32 s75, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s76, v1
+; SI-NEXT: v_readfirstlane_b32 s74, v1
; SI-NEXT: ; implicit-def: $vgpr61 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s4, s45, 24
+; SI-NEXT: s_lshr_b32 s4, s43, 24
; SI-NEXT: v_writelane_b32 v62, s4, 17
-; SI-NEXT: s_lshr_b32 s4, s47, 24
+; SI-NEXT: s_lshr_b32 s4, s45, 24
; SI-NEXT: v_writelane_b32 v62, s4, 16
-; SI-NEXT: s_lshr_b32 s4, s57, 24
+; SI-NEXT: s_lshr_b32 s4, s47, 24
; SI-NEXT: v_writelane_b32 v62, s4, 15
-; SI-NEXT: s_lshr_b32 s4, s57, 16
+; SI-NEXT: s_lshr_b32 s4, s47, 16
; SI-NEXT: v_writelane_b32 v62, s4, 14
-; SI-NEXT: s_lshr_b32 s4, s59, 24
+; SI-NEXT: s_lshr_b32 s4, s57, 24
; SI-NEXT: v_writelane_b32 v62, s4, 13
-; SI-NEXT: s_lshr_b32 s4, s59, 16
+; SI-NEXT: s_lshr_b32 s4, s57, 16
; SI-NEXT: v_writelane_b32 v62, s4, 12
-; SI-NEXT: s_lshr_b32 s4, s59, 8
+; SI-NEXT: s_lshr_b32 s4, s57, 8
; SI-NEXT: v_writelane_b32 v62, s4, 11
-; SI-NEXT: s_lshr_b32 s4, s61, 24
+; SI-NEXT: s_lshr_b32 s4, s59, 24
; SI-NEXT: v_writelane_b32 v62, s4, 10
-; SI-NEXT: s_lshr_b32 s4, s61, 16
+; SI-NEXT: s_lshr_b32 s4, s59, 16
; SI-NEXT: v_writelane_b32 v62, s4, 9
-; SI-NEXT: s_lshr_b32 s4, s61, 8
+; SI-NEXT: s_lshr_b32 s4, s59, 8
; SI-NEXT: v_writelane_b32 v62, s4, 8
-; SI-NEXT: s_lshr_b32 s4, s63, 24
+; SI-NEXT: s_lshr_b32 s4, s61, 24
; SI-NEXT: v_writelane_b32 v62, s4, 7
-; SI-NEXT: s_lshr_b32 s4, s63, 16
+; SI-NEXT: s_lshr_b32 s4, s61, 16
; SI-NEXT: v_writelane_b32 v62, s4, 6
-; SI-NEXT: s_lshr_b32 s4, s63, 8
+; SI-NEXT: s_lshr_b32 s4, s61, 8
; SI-NEXT: v_writelane_b32 v62, s4, 5
-; SI-NEXT: s_lshr_b32 s4, s73, 24
+; SI-NEXT: s_lshr_b32 s4, s63, 24
; SI-NEXT: v_writelane_b32 v62, s4, 4
-; SI-NEXT: s_lshr_b32 s4, s73, 16
+; SI-NEXT: s_lshr_b32 s4, s63, 16
; SI-NEXT: v_writelane_b32 v62, s4, 3
-; SI-NEXT: s_lshr_b32 s4, s73, 8
+; SI-NEXT: s_lshr_b32 s4, s63, 8
; SI-NEXT: v_writelane_b32 v62, s4, 2
-; SI-NEXT: s_lshr_b32 s4, s75, 24
+; SI-NEXT: s_lshr_b32 s4, s73, 24
; SI-NEXT: v_writelane_b32 v62, s4, 1
-; SI-NEXT: s_lshr_b32 s4, s75, 16
+; SI-NEXT: s_lshr_b32 s4, s73, 16
; SI-NEXT: v_writelane_b32 v62, s4, 0
-; SI-NEXT: s_lshr_b32 s4, s75, 8
+; SI-NEXT: s_lshr_b32 s4, s73, 8
; SI-NEXT: v_writelane_b32 v61, s4, 63
-; SI-NEXT: s_lshr_b32 s4, s77, 24
+; SI-NEXT: s_lshr_b32 s4, s75, 24
; SI-NEXT: v_writelane_b32 v61, s4, 62
-; SI-NEXT: s_lshr_b32 s4, s77, 16
+; SI-NEXT: s_lshr_b32 s4, s75, 16
; SI-NEXT: v_writelane_b32 v61, s4, 61
-; SI-NEXT: s_lshr_b32 s4, s77, 8
+; SI-NEXT: s_lshr_b32 s4, s75, 8
; SI-NEXT: v_writelane_b32 v61, s4, 60
; SI-NEXT: s_lshr_b32 s4, s29, 24
; SI-NEXT: v_writelane_b32 v61, s4, 59
@@ -43026,324 +43360,114 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v61, s4, 41
; SI-NEXT: s_lshr_b32 s4, s17, 8
; SI-NEXT: v_writelane_b32 v61, s4, 40
-; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 24
; SI-NEXT: v_writelane_b32 v61, s4, 38
; SI-NEXT: v_writelane_b32 v61, s5, 39
-; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 16
; SI-NEXT: v_writelane_b32 v61, s4, 36
; SI-NEXT: v_writelane_b32 v61, s5, 37
-; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 8
; SI-NEXT: v_writelane_b32 v61, s4, 34
; SI-NEXT: v_writelane_b32 v61, s5, 35
-; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 24
; SI-NEXT: v_writelane_b32 v61, s4, 32
; SI-NEXT: v_writelane_b32 v61, s5, 33
-; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 16
; SI-NEXT: v_writelane_b32 v61, s4, 30
; SI-NEXT: v_writelane_b32 v61, s5, 31
-; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[44:45], 8
; SI-NEXT: v_writelane_b32 v61, s4, 28
; SI-NEXT: v_writelane_b32 v61, s5, 29
-; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 24
; SI-NEXT: v_writelane_b32 v61, s4, 26
; SI-NEXT: v_writelane_b32 v61, s5, 27
-; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 16
; SI-NEXT: v_writelane_b32 v61, s4, 24
; SI-NEXT: v_writelane_b32 v61, s5, 25
-; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[46:47], 8
; SI-NEXT: v_writelane_b32 v61, s4, 22
; SI-NEXT: v_writelane_b32 v61, s5, 23
-; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 24
; SI-NEXT: v_writelane_b32 v61, s4, 20
; SI-NEXT: v_writelane_b32 v61, s5, 21
-; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 16
; SI-NEXT: v_writelane_b32 v61, s4, 18
; SI-NEXT: v_writelane_b32 v61, s5, 19
-; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 8
; SI-NEXT: v_writelane_b32 v61, s4, 16
; SI-NEXT: v_writelane_b32 v61, s5, 17
-; SI-NEXT: s_lshr_b64 s[4:5], s[60:61], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 24
; SI-NEXT: v_writelane_b32 v61, s4, 14
; SI-NEXT: v_writelane_b32 v61, s5, 15
-; SI-NEXT: s_lshr_b64 s[4:5], s[60:61], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 16
; SI-NEXT: v_writelane_b32 v61, s4, 12
; SI-NEXT: v_writelane_b32 v61, s5, 13
-; SI-NEXT: s_lshr_b64 s[4:5], s[62:63], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[58:59], 8
; SI-NEXT: v_writelane_b32 v61, s4, 10
; SI-NEXT: v_writelane_b32 v61, s5, 11
-; SI-NEXT: s_lshr_b64 s[4:5], s[72:73], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[60:61], 8
; SI-NEXT: v_writelane_b32 v61, s4, 8
; SI-NEXT: v_writelane_b32 v61, s5, 9
-; SI-NEXT: s_lshr_b64 s[4:5], s[72:73], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[62:63], 24
; SI-NEXT: v_writelane_b32 v61, s4, 6
; SI-NEXT: v_writelane_b32 v61, s5, 7
-; SI-NEXT: s_lshr_b64 s[4:5], s[74:75], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[62:63], 8
; SI-NEXT: v_writelane_b32 v61, s4, 4
; SI-NEXT: v_writelane_b32 v61, s5, 5
-; SI-NEXT: s_lshr_b64 s[4:5], s[76:77], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[72:73], 8
; SI-NEXT: v_writelane_b32 v61, s4, 2
; SI-NEXT: v_writelane_b32 v61, s5, 3
-; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[74:75], 8
; SI-NEXT: v_writelane_b32 v61, s4, 0
-; SI-NEXT: s_lshr_b32 s79, s45, 16
-; SI-NEXT: s_lshr_b32 s89, s45, 8
-; SI-NEXT: s_lshr_b32 s91, s47, 16
-; SI-NEXT: s_lshr_b32 s78, s47, 8
-; SI-NEXT: s_lshr_b32 s88, s57, 8
+; SI-NEXT: s_lshr_b32 s79, s43, 16
+; SI-NEXT: s_lshr_b32 s89, s43, 8
+; SI-NEXT: s_lshr_b32 s91, s45, 16
+; SI-NEXT: s_lshr_b32 s78, s45, 8
+; SI-NEXT: s_lshr_b32 s88, s47, 8
; SI-NEXT: s_lshr_b32 s90, s17, 24
-; SI-NEXT: s_lshr_b64 s[92:93], s[60:61], 16
-; SI-NEXT: s_lshr_b64 s[94:95], s[62:63], 24
+; SI-NEXT: s_lshr_b64 s[92:93], s[60:61], 24
+; SI-NEXT: s_lshr_b64 s[94:95], s[60:61], 16
; SI-NEXT: s_lshr_b64 s[30:31], s[62:63], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[72:73], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[74:75], 24
-; SI-NEXT: s_lshr_b64 s[38:39], s[74:75], 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[76:77], 24
-; SI-NEXT: s_lshr_b64 s[50:51], s[76:77], 16
-; SI-NEXT: s_lshr_b64 s[52:53], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[54:55], s[28:29], 16
-; SI-NEXT: s_lshr_b64 s[64:65], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[66:67], s[26:27], 24
-; SI-NEXT: s_lshr_b64 s[68:69], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[70:71], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[80:81], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[82:83], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[84:85], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[86:87], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[96:97], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[98:99], s[22:23], 8
-; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[42:43], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[72:73], 24
+; SI-NEXT: s_lshr_b64 s[36:37], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[74:75], 24
+; SI-NEXT: s_lshr_b64 s[48:49], s[74:75], 16
; SI-NEXT: v_writelane_b32 v61, s5, 1
-; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[50:51], s[28:29], 24
+; SI-NEXT: s_lshr_b64 s[52:53], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[28:29], 8
+; SI-NEXT: s_lshr_b64 s[64:65], s[26:27], 24
+; SI-NEXT: s_lshr_b64 s[66:67], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[26:27], 8
+; SI-NEXT: s_lshr_b64 s[70:71], s[24:25], 24
+; SI-NEXT: s_lshr_b64 s[80:81], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[82:83], s[24:25], 8
+; SI-NEXT: s_lshr_b64 s[84:85], s[22:23], 24
+; SI-NEXT: s_lshr_b64 s[86:87], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[22:23], 8
+; SI-NEXT: s_lshr_b64 s[98:99], s[20:21], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 8
+; SI-NEXT: s_lshr_b64 vcc, s[18:19], 24
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v1, s60, 1.0
-; SI-NEXT: v_add_f32_e64 v22, s26, 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s26, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e64 v1, s58, 1.0
-; SI-NEXT: v_add_f32_e64 v20, s28, 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s28, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e64 v1, s56, 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s40, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e64 v1, s46, 1.0
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e64 v1, s45, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s44, 1.0
-; SI-NEXT: v_add_f32_e64 v13, s73, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s44, v2
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v1
-; SI-NEXT: v_add_f32_e64 v15, s75, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v13
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v15
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
-; SI-NEXT: v_add_f32_e64 v17, s77, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v15
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v17
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
-; SI-NEXT: v_add_f32_e64 v19, s29, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v17
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v19
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v19
-; SI-NEXT: v_add_f32_e64 v21, s27, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v19
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v21
-; SI-NEXT: v_readfirstlane_b32 s45, v1
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v21
-; SI-NEXT: v_add_f32_e64 v37, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v38, s16, 1.0
-; SI-NEXT: v_add_f32_e64 v32, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v35, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v28, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v31, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v25, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v26, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v23, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s24, 1.0
-; SI-NEXT: v_add_f32_e64 v18, s76, 1.0
-; SI-NEXT: v_add_f32_e64 v16, s74, 1.0
-; SI-NEXT: v_add_f32_e64 v14, s72, 1.0
-; SI-NEXT: v_add_f32_e64 v11, s63, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s62, 1.0
-; SI-NEXT: v_add_f32_e64 v9, s61, 1.0
-; SI-NEXT: v_add_f32_e64 v7, s59, 1.0
-; SI-NEXT: v_add_f32_e64 v5, s57, 1.0
-; SI-NEXT: v_add_f32_e64 v3, s47, 1.0
-; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v21
-; SI-NEXT: v_readfirstlane_b32 s16, v38
-; SI-NEXT: v_readfirstlane_b32 s17, v37
-; SI-NEXT: v_readfirstlane_b32 s12, v35
-; SI-NEXT: v_readfirstlane_b32 s13, v32
-; SI-NEXT: v_readfirstlane_b32 s6, v31
-; SI-NEXT: v_readfirstlane_b32 s7, v28
-; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: v_readfirstlane_b32 s5, v25
-; SI-NEXT: v_readfirstlane_b32 s8, v24
-; SI-NEXT: v_readfirstlane_b32 s9, v23
-; SI-NEXT: v_readfirstlane_b32 s10, v22
-; SI-NEXT: v_readfirstlane_b32 s11, v21
-; SI-NEXT: v_readfirstlane_b32 s14, v20
-; SI-NEXT: v_readfirstlane_b32 s15, v19
-; SI-NEXT: v_readfirstlane_b32 s18, v18
-; SI-NEXT: v_readfirstlane_b32 s19, v17
-; SI-NEXT: v_readfirstlane_b32 s20, v16
-; SI-NEXT: v_readfirstlane_b32 s21, v15
-; SI-NEXT: v_readfirstlane_b32 s22, v14
-; SI-NEXT: v_readfirstlane_b32 s23, v13
-; SI-NEXT: v_readfirstlane_b32 s24, v12
-; SI-NEXT: v_readfirstlane_b32 s25, v11
-; SI-NEXT: v_readfirstlane_b32 s27, v9
-; SI-NEXT: v_readfirstlane_b32 s29, v7
-; SI-NEXT: v_readfirstlane_b32 s41, v5
-; SI-NEXT: v_readfirstlane_b32 s43, v3
-; SI-NEXT: s_lshr_b64 vcc, s[44:45], 24
-; SI-NEXT: v_writelane_b32 v61, s46, 36
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v23
-; SI-NEXT: v_writelane_b32 v61, s47, 37
-; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[58:59], s[42:43], 24
-; SI-NEXT: s_lshr_b64 s[60:61], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[42:43], 8
-; SI-NEXT: s_lshr_b64 s[72:73], s[40:41], 24
-; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[40:41], 8
-; SI-NEXT: s_lshr_b64 s[78:79], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[88:89], s[28:29], 16
-; SI-NEXT: s_lshr_b64 s[28:29], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; SI-NEXT: s_lshr_b64 s[92:93], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[26:27], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[94:95], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[30:31], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[24:25], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[46:47], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[22:23], s[22:23], 8
-; SI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[38:39], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[20:21], s[20:21], 8
-; SI-NEXT: s_lshr_b64 s[48:49], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[50:51], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[18:19], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[52:53], s[14:15], 24
-; SI-NEXT: s_lshr_b64 s[54:55], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[64:65], s[14:15], 8
-; SI-NEXT: s_lshr_b64 s[66:67], s[10:11], 24
-; SI-NEXT: s_lshr_b64 s[68:69], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[70:71], s[10:11], 8
-; SI-NEXT: s_lshr_b64 s[80:81], s[8:9], 24
-; SI-NEXT: s_lshr_b64 s[82:83], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[84:85], s[8:9], 8
-; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 24
-; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[98:99], s[4:5], 8
-; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 24
-; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 8
-; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 24
-; SI-NEXT: s_lshr_b64 s[10:11], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[14:15], s[12:13], 8
-; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 8
-; SI-NEXT: s_mov_b32 s16, vcc_lo
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v41, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v44, 8, v3
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v47, 8, v5
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v7
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v58, 8, v7
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v27, 8, v9
-; SI-NEXT: v_lshrrev_b32_e32 v29, 24, v11
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v11
-; SI-NEXT: v_lshrrev_b32_e32 v34, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v13
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v25
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v4, 8, v25
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v28
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v51, 8, v28
-; SI-NEXT: v_lshrrev_b32_e32 v52, 24, v32
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v53, 8, v32
-; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v37
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v37
-; SI-NEXT: v_lshrrev_b32_e32 v55, 8, v37
-; SI-NEXT: s_branch .LBB37_5
-; SI-NEXT: .LBB37_3:
+; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 8
+; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[76:77], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr91
; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: s_mov_b64 s[76:77], -1
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s8, 0
+; SI-NEXT: v_writelane_b32 v61, s90, 0
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s9, 1
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s90, 2
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s91, 3
+; SI-NEXT: v_writelane_b32 v61, s91, 1
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr90
; SI-NEXT: ; kill: killed $sgpr4
@@ -43354,9 +43478,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr96
@@ -43379,9 +43503,16 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s90, 4
+; SI-NEXT: v_writelane_b32 v61, s90, 2
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v61, s91, 3
+; SI-NEXT: ; implicit-def: $sgpr90
; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v61, s90, 4
; SI-NEXT: v_writelane_b32 v61, s91, 5
; SI-NEXT: ; implicit-def: $sgpr90
; SI-NEXT: ; kill: killed $sgpr4
@@ -43505,8 +43636,217 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: ; kill: killed $sgpr4
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f32_e64 v1, s58, 1.0
+; SI-NEXT: v_add_f32_e64 v22, s26, 1.0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s26, v1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e64 v1, s56, 1.0
+; SI-NEXT: v_add_f32_e64 v20, s28, 1.0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s28, v1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e64 v1, s46, 1.0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s40, v1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e64 v1, s44, 1.0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s44, v1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e64 v1, s43, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s42, 1.0
+; SI-NEXT: v_add_f32_e64 v13, s63, 1.0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s42, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v1
+; SI-NEXT: v_add_f32_e64 v15, s73, 1.0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v13
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v15
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; SI-NEXT: v_add_f32_e64 v17, s75, 1.0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v15
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v17
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
+; SI-NEXT: v_add_f32_e64 v19, s29, 1.0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v17
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v19
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v19
+; SI-NEXT: v_add_f32_e64 v21, s27, 1.0
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v19
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v21
+; SI-NEXT: v_readfirstlane_b32 s43, v1
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v21
+; SI-NEXT: v_add_f32_e64 v37, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v38, s16, 1.0
+; SI-NEXT: v_add_f32_e64 v32, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v35, s18, 1.0
+; SI-NEXT: v_add_f32_e64 v28, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v31, s20, 1.0
+; SI-NEXT: v_add_f32_e64 v25, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v26, s22, 1.0
+; SI-NEXT: v_add_f32_e64 v23, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v24, s24, 1.0
+; SI-NEXT: v_add_f32_e64 v18, s74, 1.0
+; SI-NEXT: v_add_f32_e64 v16, s72, 1.0
+; SI-NEXT: v_add_f32_e64 v14, s62, 1.0
+; SI-NEXT: v_add_f32_e64 v11, s61, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s60, 1.0
+; SI-NEXT: v_add_f32_e64 v9, s59, 1.0
+; SI-NEXT: v_add_f32_e64 v7, s57, 1.0
+; SI-NEXT: v_add_f32_e64 v5, s47, 1.0
+; SI-NEXT: v_add_f32_e64 v3, s45, 1.0
+; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v21
+; SI-NEXT: v_readfirstlane_b32 s16, v38
+; SI-NEXT: v_readfirstlane_b32 s17, v37
+; SI-NEXT: v_readfirstlane_b32 s12, v35
+; SI-NEXT: v_readfirstlane_b32 s13, v32
+; SI-NEXT: v_readfirstlane_b32 s6, v31
+; SI-NEXT: v_readfirstlane_b32 s7, v28
+; SI-NEXT: v_readfirstlane_b32 s4, v26
+; SI-NEXT: v_readfirstlane_b32 s5, v25
+; SI-NEXT: v_readfirstlane_b32 s8, v24
+; SI-NEXT: v_readfirstlane_b32 s9, v23
+; SI-NEXT: v_readfirstlane_b32 s10, v22
+; SI-NEXT: v_readfirstlane_b32 s11, v21
+; SI-NEXT: v_readfirstlane_b32 s14, v20
+; SI-NEXT: v_readfirstlane_b32 s15, v19
+; SI-NEXT: v_readfirstlane_b32 s18, v18
+; SI-NEXT: v_readfirstlane_b32 s19, v17
+; SI-NEXT: v_readfirstlane_b32 s20, v16
+; SI-NEXT: v_readfirstlane_b32 s21, v15
+; SI-NEXT: v_readfirstlane_b32 s22, v14
+; SI-NEXT: v_readfirstlane_b32 s23, v13
+; SI-NEXT: v_readfirstlane_b32 s24, v12
+; SI-NEXT: v_readfirstlane_b32 s25, v11
+; SI-NEXT: v_readfirstlane_b32 s27, v9
+; SI-NEXT: v_readfirstlane_b32 s29, v7
+; SI-NEXT: v_readfirstlane_b32 s41, v5
+; SI-NEXT: v_readfirstlane_b32 s45, v3
+; SI-NEXT: s_lshr_b64 vcc, s[42:43], 24
+; SI-NEXT: v_writelane_b32 v61, s46, 36
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v23
+; SI-NEXT: v_writelane_b32 v61, s47, 37
+; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 8
+; SI-NEXT: s_lshr_b64 s[58:59], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[60:61], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 8
+; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
+; SI-NEXT: s_lshr_b64 s[72:73], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 8
+; SI-NEXT: s_lshr_b64 s[76:77], s[28:29], 24
+; SI-NEXT: s_lshr_b64 s[78:79], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[28:29], s[28:29], 8
+; SI-NEXT: s_lshr_b64 s[88:89], s[26:27], 24
+; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[26:27], 8
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
+; SI-NEXT: s_lshr_b64 s[94:95], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[24:25], 8
+; SI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; SI-NEXT: s_lshr_b64 s[46:47], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[22:23], s[22:23], 8
+; SI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; SI-NEXT: s_lshr_b64 s[36:37], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[20:21], s[20:21], 8
+; SI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[48:49], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[18:19], s[18:19], 8
+; SI-NEXT: s_lshr_b64 s[50:51], s[14:15], 24
+; SI-NEXT: s_lshr_b64 s[52:53], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[14:15], 8
+; SI-NEXT: s_lshr_b64 s[64:65], s[10:11], 24
+; SI-NEXT: s_lshr_b64 s[66:67], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[68:69], s[10:11], 8
+; SI-NEXT: s_lshr_b64 s[70:71], s[8:9], 24
+; SI-NEXT: s_lshr_b64 s[80:81], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[82:83], s[8:9], 8
+; SI-NEXT: s_lshr_b64 s[84:85], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 8
+; SI-NEXT: s_lshr_b64 s[98:99], s[6:7], 24
+; SI-NEXT: s_lshr_b64 s[4:5], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 8
+; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 24
+; SI-NEXT: s_lshr_b64 s[10:11], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[12:13], 8
+; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 8
+; SI-NEXT: s_mov_b32 s16, vcc_lo
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v41, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v44, 8, v3
+; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v47, 8, v5
+; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v58, 8, v7
+; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 8, v9
+; SI-NEXT: v_lshrrev_b32_e32 v29, 24, v11
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v11
+; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v11
+; SI-NEXT: v_lshrrev_b32_e32 v34, 24, v13
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v13
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v23
+; SI-NEXT: v_lshrrev_b32_e32 v48, 24, v25
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v4, 8, v25
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v28
+; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v51, 8, v28
+; SI-NEXT: v_lshrrev_b32_e32 v52, 24, v32
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v32
+; SI-NEXT: v_lshrrev_b32_e32 v53, 8, v32
+; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v37
+; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v37
+; SI-NEXT: v_lshrrev_b32_e32 v55, 8, v37
+; SI-NEXT: s_branch .LBB37_6
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -43611,28 +43951,28 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s5, v62, 12
; SI-NEXT: v_mov_b32_e32 v57, s5
; SI-NEXT: v_readlane_b32 s5, v62, 13
-; SI-NEXT: v_mov_b32_e32 v1, s60
+; SI-NEXT: v_mov_b32_e32 v1, s58
; SI-NEXT: v_mov_b32_e32 v56, s5
; SI-NEXT: v_readlane_b32 s5, v62, 14
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s58
+; SI-NEXT: v_mov_b32_e32 v1, s56
; SI-NEXT: v_mov_b32_e32 v46, s5
; SI-NEXT: v_readlane_b32 s5, v62, 15
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s56
+; SI-NEXT: v_mov_b32_e32 v1, s46
; SI-NEXT: v_mov_b32_e32 v45, s5
; SI-NEXT: v_readlane_b32 s5, v62, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s46
+; SI-NEXT: v_mov_b32_e32 v1, s44
; SI-NEXT: v_mov_b32_e32 v42, s5
; SI-NEXT: v_readlane_b32 s5, v62, 17
+; SI-NEXT: v_mov_b32_e32 v5, s47
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v3, s47
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s44
+; SI-NEXT: v_mov_b32_e32 v1, s42
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v12, s5
@@ -43643,15 +43983,15 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v23, s25
; SI-NEXT: v_mov_b32_e32 v21, s27
; SI-NEXT: v_mov_b32_e32 v19, s29
-; SI-NEXT: v_mov_b32_e32 v17, s77
-; SI-NEXT: v_mov_b32_e32 v15, s75
-; SI-NEXT: v_mov_b32_e32 v13, s73
-; SI-NEXT: v_mov_b32_e32 v11, s63
-; SI-NEXT: v_mov_b32_e32 v9, s61
-; SI-NEXT: v_mov_b32_e32 v7, s59
+; SI-NEXT: v_mov_b32_e32 v17, s75
+; SI-NEXT: v_mov_b32_e32 v15, s73
+; SI-NEXT: v_mov_b32_e32 v13, s63
+; SI-NEXT: v_mov_b32_e32 v11, s61
+; SI-NEXT: v_mov_b32_e32 v9, s59
+; SI-NEXT: v_mov_b32_e32 v3, s45
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s45
+; SI-NEXT: v_mov_b32_e32 v1, s43
; SI-NEXT: v_mov_b32_e32 v10, s90
; SI-NEXT: v_mov_b32_e32 v47, s88
; SI-NEXT: v_mov_b32_e32 v44, s78
@@ -43660,10 +44000,10 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v40, s79
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s62
-; SI-NEXT: v_mov_b32_e32 v14, s72
-; SI-NEXT: v_mov_b32_e32 v16, s74
-; SI-NEXT: v_mov_b32_e32 v18, s76
+; SI-NEXT: v_mov_b32_e32 v12, s60
+; SI-NEXT: v_mov_b32_e32 v14, s62
+; SI-NEXT: v_mov_b32_e32 v16, s72
+; SI-NEXT: v_mov_b32_e32 v18, s74
; SI-NEXT: v_mov_b32_e32 v20, s28
; SI-NEXT: v_mov_b32_e32 v22, s26
; SI-NEXT: v_mov_b32_e32 v24, s24
@@ -43671,55 +44011,55 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v31, s20
; SI-NEXT: v_mov_b32_e32 v35, s18
; SI-NEXT: s_mov_b32 s16, s46
-; SI-NEXT: v_readlane_b32 s44, v61, 34
+; SI-NEXT: v_readlane_b32 s42, v61, 34
; SI-NEXT: v_readlane_b32 s58, v61, 32
; SI-NEXT: v_readlane_b32 s60, v61, 30
-; SI-NEXT: v_readlane_b32 s62, v61, 28
-; SI-NEXT: v_readlane_b32 s72, v61, 26
-; SI-NEXT: v_readlane_b32 s74, v61, 24
-; SI-NEXT: v_readlane_b32 s76, v61, 22
-; SI-NEXT: v_readlane_b32 s78, v61, 20
-; SI-NEXT: v_readlane_b32 s88, v61, 18
+; SI-NEXT: v_readlane_b32 s44, v61, 28
+; SI-NEXT: v_readlane_b32 s62, v61, 26
+; SI-NEXT: v_readlane_b32 s72, v61, 24
+; SI-NEXT: v_readlane_b32 s74, v61, 22
+; SI-NEXT: v_readlane_b32 s76, v61, 20
+; SI-NEXT: v_readlane_b32 s78, v61, 18
; SI-NEXT: v_readlane_b32 s28, v61, 16
-; SI-NEXT: v_readlane_b32 s90, v61, 14
-; SI-NEXT: v_readlane_b32 s26, v61, 12
-; SI-NEXT: v_readlane_b32 s24, v61, 10
-; SI-NEXT: v_readlane_b32 s22, v61, 6
-; SI-NEXT: s_mov_b32 s46, s34
-; SI-NEXT: v_readlane_b32 s34, v61, 8
-; SI-NEXT: v_readlane_b32 s20, v61, 4
-; SI-NEXT: v_readlane_b32 s18, v61, 2
-; SI-NEXT: s_mov_b32 s56, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 0
+; SI-NEXT: v_readlane_b32 s88, v61, 14
+; SI-NEXT: v_readlane_b32 s90, v61, 12
+; SI-NEXT: v_readlane_b32 s26, v61, 10
+; SI-NEXT: v_readlane_b32 s24, v61, 8
+; SI-NEXT: v_readlane_b32 s22, v61, 4
+; SI-NEXT: s_mov_b32 s46, s30
+; SI-NEXT: v_readlane_b32 s30, v61, 6
+; SI-NEXT: v_readlane_b32 s20, v61, 2
+; SI-NEXT: v_readlane_b32 s18, v61, 0
; SI-NEXT: v_mov_b32_e32 v37, s17
-; SI-NEXT: v_mov_b32_e32 v5, s57
+; SI-NEXT: v_mov_b32_e32 v7, s57
; SI-NEXT: v_readlane_b32 s47, v61, 39
-; SI-NEXT: v_readlane_b32 s45, v61, 35
+; SI-NEXT: v_readlane_b32 s43, v61, 35
; SI-NEXT: v_readlane_b32 s59, v61, 33
; SI-NEXT: v_readlane_b32 s61, v61, 31
-; SI-NEXT: v_readlane_b32 s63, v61, 29
-; SI-NEXT: v_readlane_b32 s73, v61, 27
-; SI-NEXT: v_readlane_b32 s75, v61, 25
-; SI-NEXT: v_readlane_b32 s77, v61, 23
-; SI-NEXT: v_readlane_b32 s79, v61, 21
-; SI-NEXT: v_readlane_b32 s89, v61, 19
+; SI-NEXT: v_readlane_b32 s45, v61, 29
+; SI-NEXT: v_readlane_b32 s63, v61, 27
+; SI-NEXT: v_readlane_b32 s73, v61, 25
+; SI-NEXT: v_readlane_b32 s75, v61, 23
+; SI-NEXT: v_readlane_b32 s77, v61, 21
+; SI-NEXT: v_readlane_b32 s79, v61, 19
; SI-NEXT: v_readlane_b32 s29, v61, 17
-; SI-NEXT: v_readlane_b32 s91, v61, 15
-; SI-NEXT: v_readlane_b32 s27, v61, 13
-; SI-NEXT: v_readlane_b32 s25, v61, 11
-; SI-NEXT: v_readlane_b32 s23, v61, 7
-; SI-NEXT: v_readlane_b32 s35, v61, 9
-; SI-NEXT: v_readlane_b32 s21, v61, 5
-; SI-NEXT: v_readlane_b32 s19, v61, 3
-; SI-NEXT: v_readlane_b32 s9, v61, 1
-; SI-NEXT: .LBB37_5: ; %end
+; SI-NEXT: v_readlane_b32 s89, v61, 15
+; SI-NEXT: v_readlane_b32 s91, v61, 13
+; SI-NEXT: v_readlane_b32 s27, v61, 11
+; SI-NEXT: v_readlane_b32 s25, v61, 9
+; SI-NEXT: v_readlane_b32 s23, v61, 5
+; SI-NEXT: v_readlane_b32 s31, v61, 7
+; SI-NEXT: v_readlane_b32 s21, v61, 3
+; SI-NEXT: v_readlane_b32 s19, v61, 1
+; SI-NEXT: s_mov_b32 s56, vcc_lo
+; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v38, 0xff, v38
-; SI-NEXT: s_lshl_b32 s5, s4, 8
+; SI-NEXT: s_lshl_b32 s5, s6, 8
; SI-NEXT: v_or_b32_e32 v38, s5, v38
-; SI-NEXT: s_and_b32 s5, s6, 0xff
+; SI-NEXT: s_and_b32 s5, s12, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s12, 24
+; SI-NEXT: s_lshl_b32 s7, s14, 24
; SI-NEXT: v_and_b32_e32 v38, 0xffff, v38
; SI-NEXT: s_or_b32 s5, s7, s5
; SI-NEXT: v_or_b32_e32 v38, s5, v38
@@ -43738,7 +44078,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v10, v37, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v10, 0xff, v35
-; SI-NEXT: s_lshl_b32 s5, s14, 8
+; SI-NEXT: s_lshl_b32 s5, s8, 8
; SI-NEXT: v_or_b32_e32 v10, s5, v10
; SI-NEXT: s_and_b32 s5, s10, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
@@ -43762,11 +44102,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v8, v10, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xff, v31
-; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: s_and_b32 s4, s42, 0xff
+; SI-NEXT: s_lshl_b32 s5, s40, 8
+; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: v_or_b32_e32 v8, s5, v8
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s40, 24
+; SI-NEXT: s_lshl_b32 s5, s98, 24
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v8, s4, v8
@@ -43786,11 +44126,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v6, v8, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v6, 0xff, v26
-; SI-NEXT: s_lshl_b32 s4, s98, 8
+; SI-NEXT: s_lshl_b32 s4, s96, 8
; SI-NEXT: v_or_b32_e32 v6, s4, v6
-; SI-NEXT: s_and_b32 s4, s96, 0xff
+; SI-NEXT: s_and_b32 s4, s86, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s86, 24
+; SI-NEXT: s_lshl_b32 s5, s84, 24
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v6, s4, v6
@@ -43810,11 +44150,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v4, v6, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v4, 0xff, v24
-; SI-NEXT: s_lshl_b32 s4, s84, 8
+; SI-NEXT: s_lshl_b32 s4, s82, 8
; SI-NEXT: v_or_b32_e32 v4, s4, v4
-; SI-NEXT: s_and_b32 s4, s82, 0xff
+; SI-NEXT: s_and_b32 s4, s80, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s80, 24
+; SI-NEXT: s_lshl_b32 s5, s70, 24
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v4, s4, v4
@@ -43828,8 +44168,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: s_lshl_b32 s4, s70, 8
-; SI-NEXT: s_lshl_b32 s5, s66, 24
+; SI-NEXT: s_lshl_b32 s4, s68, 8
+; SI-NEXT: s_lshl_b32 s5, s64, 24
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_readlane_b32 s99, v63, 33
; SI-NEXT: v_readlane_b32 s98, v63, 32
@@ -43846,9 +44186,10 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s71, v63, 21
; SI-NEXT: v_readlane_b32 s70, v63, 20
; SI-NEXT: v_readlane_b32 s69, v63, 19
+; SI-NEXT: v_readlane_b32 s68, v63, 18
; SI-NEXT: v_readlane_b32 s67, v63, 17
-; SI-NEXT: v_readlane_b32 s66, v63, 16
; SI-NEXT: v_readlane_b32 s65, v63, 15
+; SI-NEXT: v_readlane_b32 s64, v63, 14
; SI-NEXT: v_readlane_b32 s55, v63, 13
; SI-NEXT: v_readlane_b32 s53, v63, 11
; SI-NEXT: v_readlane_b32 s51, v63, 9
@@ -43865,7 +44206,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v22
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s68, 0xff
+; SI-NEXT: s_and_b32 s4, s66, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -43875,11 +44216,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v21
-; SI-NEXT: s_lshl_b32 s4, s64, 8
-; SI-NEXT: s_lshl_b32 s5, s52, 24
-; SI-NEXT: v_readlane_b32 s68, v63, 18
-; SI-NEXT: v_readlane_b32 s64, v63, 14
-; SI-NEXT: v_readlane_b32 s52, v63, 10
+; SI-NEXT: s_lshl_b32 s4, s54, 8
+; SI-NEXT: s_lshl_b32 s5, s50, 24
+; SI-NEXT: v_readlane_b32 s66, v63, 16
+; SI-NEXT: v_readlane_b32 s54, v63, 12
+; SI-NEXT: v_readlane_b32 s50, v63, 8
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
@@ -43898,7 +44239,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s54, 0xff
+; SI-NEXT: s_and_b32 s4, s52, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -43910,9 +44251,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v19
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s18, 8
-; SI-NEXT: s_lshl_b32 s5, s48, 24
-; SI-NEXT: v_readlane_b32 s54, v63, 12
-; SI-NEXT: v_readlane_b32 s48, v63, 6
+; SI-NEXT: s_lshl_b32 s5, s38, 24
+; SI-NEXT: v_readlane_b32 s52, v63, 10
+; SI-NEXT: v_readlane_b32 s38, v63, 4
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -43930,7 +44271,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v18
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s50, 0xff
+; SI-NEXT: s_and_b32 s4, s48, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -43942,9 +44283,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v17
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s20, 8
-; SI-NEXT: s_lshl_b32 s5, s36, 24
-; SI-NEXT: v_readlane_b32 s50, v63, 8
-; SI-NEXT: v_readlane_b32 s36, v63, 2
+; SI-NEXT: s_lshl_b32 s5, s34, 24
+; SI-NEXT: v_readlane_b32 s48, v63, 6
+; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -43962,7 +44303,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v16
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s38, 0xff
+; SI-NEXT: s_and_b32 s4, s36, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -43974,9 +44315,10 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v15
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s22, 8
-; SI-NEXT: s_lshl_b32 s5, s34, 24
-; SI-NEXT: v_readlane_b32 s38, v63, 4
-; SI-NEXT: v_readlane_b32 s34, v63, 0
+; SI-NEXT: s_lshl_b32 s5, s30, 24
+; SI-NEXT: v_readlane_b32 s30, v63, 34
+; SI-NEXT: v_readlane_b32 s31, v63, 35
+; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -44006,7 +44348,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v2, 0xff, v13
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v34
; SI-NEXT: s_lshl_b32 s4, s24, 8
-; SI-NEXT: s_lshl_b32 s5, s94, 24
+; SI-NEXT: s_lshl_b32 s5, s92, 24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -44020,7 +44362,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v12
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s30, 0xff
+; SI-NEXT: s_and_b32 s4, s94, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44042,15 +44384,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s26, 8
-; SI-NEXT: s_lshl_b32 s5, s90, 24
+; SI-NEXT: s_lshl_b32 s5, s88, 24
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x58, v0
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v59
-; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s92, 0xff
+; SI-NEXT: s_and_b32 s4, s90, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44070,13 +44410,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s28, 8
-; SI-NEXT: s_lshl_b32 s5, s78, 24
+; SI-NEXT: s_lshl_b32 s5, s76, 24
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x60, v0
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v56
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s88, 0xff
+; SI-NEXT: s_and_b32 s4, s78, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44095,13 +44435,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s4, s76, 8
-; SI-NEXT: s_lshl_b32 s5, s72, 24
+; SI-NEXT: s_lshl_b32 s4, s74, 8
+; SI-NEXT: s_lshl_b32 s5, s62, 24
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x68, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s74, 0xff
+; SI-NEXT: s_and_b32 s4, s72, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44121,7 +44461,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s4, s62, 8
+; SI-NEXT: s_lshl_b32 s4, s44, 8
; SI-NEXT: s_lshl_b32 s5, s58, 24
; SI-NEXT: v_add_i32_e32 v4, vcc, 0x70, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -44147,7 +44487,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s4, s44, 8
+; SI-NEXT: s_lshl_b32 s4, s42, 8
; SI-NEXT: v_add_i32_e32 v3, vcc, 0x78, v0
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -44269,7 +44609,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_cmp_lg_u32 s44, 0
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
; VI-NEXT: v_writelane_b32 v62, s46, 57
@@ -44420,13 +44760,175 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s47, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s47, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s5, 1.0
; VI-NEXT: v_add_f32_e64 v1, s4, 1.0
; VI-NEXT: v_lshrrev_b64 v[33:34], 24, v[1:2]
@@ -44633,164 +45135,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v60, 8, v32
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v31
; VI-NEXT: v_lshrrev_b32_e32 v61, 8, v31
-; VI-NEXT: s_branch .LBB37_5
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: s_branch .LBB37_6
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v53, s46
; VI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -45027,9 +45373,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v55, s82
; VI-NEXT: v_mov_b32_e32 v36, s81
; VI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v39, s36
-; VI-NEXT: v_mov_b32_e32 v40, s38
-; VI-NEXT: v_mov_b32_e32 v41, s48
+; VI-NEXT: v_mov_b32_e32 v39, s34
+; VI-NEXT: v_mov_b32_e32 v40, s36
+; VI-NEXT: v_mov_b32_e32 v41, s38
; VI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v53, s62
@@ -45053,9 +45399,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v53, s90
; VI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v53, s30
-; VI-NEXT: v_mov_b32_e32 v54, s34
-; VI-NEXT: .LBB37_5: ; %end
+; VI-NEXT: v_mov_b32_e32 v53, vcc_lo
+; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: .LBB37_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v33, v33, v41, s4
; VI-NEXT: v_perm_b32 v31, v31, v61, s4
@@ -45502,7 +45848,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_cmp_lg_u32 s44, 0
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
; GFX9-NEXT: v_writelane_b32 v62, s46, 49
@@ -45647,11 +45993,165 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s5, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s4, 1.0
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2]
@@ -45867,156 +46367,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v19, 8, v49
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v48
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 8, v48
-; GFX9-NEXT: s_branch .LBB37_5
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: s_branch .LBB37_6
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v53, s46
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -46250,9 +46602,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v56, s85
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v39, s94
-; GFX9-NEXT: v_mov_b32_e32 v40, s30
-; GFX9-NEXT: v_mov_b32_e32 v41, s34
-; GFX9-NEXT: v_mov_b32_e32 v42, s36
+; GFX9-NEXT: v_mov_b32_e32 v40, vcc_lo
+; GFX9-NEXT: v_mov_b32_e32 v41, s30
+; GFX9-NEXT: v_mov_b32_e32 v42, s34
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
@@ -46282,7 +46634,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v53, s90
; GFX9-NEXT: v_mov_b32_e32 v54, s92
-; GFX9-NEXT: .LBB37_5: ; %end
+; GFX9-NEXT: .LBB37_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, v32, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
@@ -46732,13 +47084,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s30, 0
; GFX11-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB37_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s5, 24
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s27, 24
+; GFX11-NEXT: s_lshr_b32 s31, s27, 24
; GFX11-NEXT: v_writelane_b32 v77, s42, 7
; GFX11-NEXT: s_lshr_b32 s42, s5, 16
; GFX11-NEXT: s_lshr_b32 s34, s27, 16
@@ -46820,7 +47172,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_writelane_b32 v76, s42, 23
; GFX11-NEXT: s_lshr_b32 s42, s11, 8
; GFX11-NEXT: s_lshr_b64 s[94:95], s[28:29], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[26:27], 24
+; GFX11-NEXT: s_lshr_b64 vcc, s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[62:63], s[24:25], 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 22
; GFX11-NEXT: s_lshr_b32 s42, s10, 16
@@ -46883,141 +47235,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_lshr_b32 s42, s28, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 0
; GFX11-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v12, s15, 1.0
-; GFX11-NEXT: v_add_f32_e64 v11, s14, 1.0
-; GFX11-NEXT: v_add_f32_e64 v14, s41, 1.0
-; GFX11-NEXT: v_add_f32_e64 v13, s40, 1.0
-; GFX11-NEXT: v_add_f32_e64 v16, s29, 1.0
-; GFX11-NEXT: v_add_f32_e64 v15, s28, 1.0
-; GFX11-NEXT: v_add_f32_e64 v26, s19, 1.0
-; GFX11-NEXT: v_add_f32_e64 v25, s18, 1.0
-; GFX11-NEXT: v_add_f32_e64 v18, s27, 1.0
-; GFX11-NEXT: v_add_f32_e64 v17, s26, 1.0
-; GFX11-NEXT: v_add_f32_e64 v4, s7, 1.0
-; GFX11-NEXT: v_add_f32_e64 v3, s6, 1.0
-; GFX11-NEXT: v_add_f32_e64 v28, s17, 1.0
-; GFX11-NEXT: v_add_f32_e64 v27, s16, 1.0
-; GFX11-NEXT: v_add_f32_e64 v20, s25, 1.0
-; GFX11-NEXT: v_add_f32_e64 v19, s24, 1.0
-; GFX11-NEXT: v_add_f32_e64 v6, s9, 1.0
-; GFX11-NEXT: v_add_f32_e64 v5, s8, 1.0
-; GFX11-NEXT: v_add_f32_e64 v32, s3, 1.0
-; GFX11-NEXT: v_add_f32_e64 v31, s2, 1.0
-; GFX11-NEXT: v_add_f32_e64 v22, s23, 1.0
-; GFX11-NEXT: v_add_f32_e64 v21, s22, 1.0
-; GFX11-NEXT: v_add_f32_e64 v8, s11, 1.0
-; GFX11-NEXT: v_add_f32_e64 v7, s10, 1.0
-; GFX11-NEXT: v_lshrrev_b64 v[48:49], 24, v[11:12]
-; GFX11-NEXT: v_add_f32_e64 v34, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v33, s0, 1.0
-; GFX11-NEXT: v_add_f32_e64 v24, s21, 1.0
-; GFX11-NEXT: v_add_f32_e64 v23, s20, 1.0
-; GFX11-NEXT: v_add_f32_e64 v10, s13, 1.0
-; GFX11-NEXT: v_add_f32_e64 v2, s5, 1.0
-; GFX11-NEXT: v_add_f32_e64 v1, s4, 1.0
-; GFX11-NEXT: v_add_f32_e64 v9, s12, 1.0
-; GFX11-NEXT: v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-NEXT: v_lshrrev_b64 v[50:51], 24, v[15:16]
-; GFX11-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
-; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[25:26]
-; GFX11-NEXT: v_lshrrev_b64 v[36:37], 24, v[5:6]
-; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[19:20]
-; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[27:28]
-; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[7:8]
-; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[21:22]
-; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[31:32]
-; GFX11-NEXT: v_lshrrev_b64 v[29:30], 24, v[1:2]
-; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[9:10]
-; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[23:24]
-; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[33:34]
-; GFX11-NEXT: v_lshrrev_b32_e32 v70, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v69, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v39, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v83, 24, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v81, 16, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v55, 8, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v82, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v86, 24, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v80, 8, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v98, 16, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v96, 8, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v100, 24, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v99, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v101, 16, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v97, 8, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v112, 24, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v102, 8, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v116, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v114, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v118, 24, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v117, 16, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v113, 8, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v119, 16, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v115, 8, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v129, 16, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v134, 16, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v144, 24, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v135, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v131, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v133, 8, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v147, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v146, 8, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v151, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v150, 8, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v163, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v160, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v166, 24, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v165, 16, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v164, 8, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v177, 16, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v180, 24, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v176, 8, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v178, 8, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v41, 24, v26
-; GFX11-NEXT: v_lshrrev_b32_e32 v40, 16, v26
-; GFX11-NEXT: v_lshrrev_b32_e32 v182, 8, v26
-; GFX11-NEXT: v_lshrrev_b32_e32 v43, 16, v25
-; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v25
-; GFX11-NEXT: v_lshrrev_b32_e32 v45, 24, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v44, 16, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v42, 8, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v47, 16, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v46, 8, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v59, 24, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v58, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v56, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v60, 16, v31
-; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v31
-; GFX11-NEXT: v_lshrrev_b32_e32 v62, 24, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v61, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v63, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v73, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v72, 8, v33
-; GFX11-NEXT: s_branch .LBB37_5
-; GFX11-NEXT: .LBB37_3:
+; GFX11-NEXT: s_branch .LBB37_3
+; GFX11-NEXT: .LBB37_2:
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: s_mov_b32 s30, -1
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr104
@@ -47066,8 +47288,8 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: ; implicit-def: $sgpr35
; GFX11-NEXT: ; implicit-def: $sgpr36
; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $vcc_lo
; GFX11-NEXT: ; implicit-def: $sgpr94
; GFX11-NEXT: ; implicit-def: $sgpr92
; GFX11-NEXT: ; implicit-def: $sgpr90
@@ -47152,8 +47374,143 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: s_branch .LBB37_2
-; GFX11-NEXT: .LBB37_4:
+; GFX11-NEXT: .LBB37_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s43, s30, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_add_f32_e64 v12, s15, 1.0
+; GFX11-NEXT: v_add_f32_e64 v11, s14, 1.0
+; GFX11-NEXT: v_add_f32_e64 v14, s41, 1.0
+; GFX11-NEXT: v_add_f32_e64 v13, s40, 1.0
+; GFX11-NEXT: v_add_f32_e64 v16, s29, 1.0
+; GFX11-NEXT: v_add_f32_e64 v15, s28, 1.0
+; GFX11-NEXT: v_add_f32_e64 v26, s19, 1.0
+; GFX11-NEXT: v_add_f32_e64 v25, s18, 1.0
+; GFX11-NEXT: v_add_f32_e64 v18, s27, 1.0
+; GFX11-NEXT: v_add_f32_e64 v17, s26, 1.0
+; GFX11-NEXT: v_add_f32_e64 v4, s7, 1.0
+; GFX11-NEXT: v_add_f32_e64 v3, s6, 1.0
+; GFX11-NEXT: v_add_f32_e64 v28, s17, 1.0
+; GFX11-NEXT: v_add_f32_e64 v27, s16, 1.0
+; GFX11-NEXT: v_add_f32_e64 v20, s25, 1.0
+; GFX11-NEXT: v_add_f32_e64 v19, s24, 1.0
+; GFX11-NEXT: v_add_f32_e64 v6, s9, 1.0
+; GFX11-NEXT: v_add_f32_e64 v5, s8, 1.0
+; GFX11-NEXT: v_add_f32_e64 v32, s3, 1.0
+; GFX11-NEXT: v_add_f32_e64 v31, s2, 1.0
+; GFX11-NEXT: v_add_f32_e64 v22, s23, 1.0
+; GFX11-NEXT: v_add_f32_e64 v21, s22, 1.0
+; GFX11-NEXT: v_add_f32_e64 v8, s11, 1.0
+; GFX11-NEXT: v_add_f32_e64 v7, s10, 1.0
+; GFX11-NEXT: v_lshrrev_b64 v[48:49], 24, v[11:12]
+; GFX11-NEXT: v_add_f32_e64 v34, s1, 1.0
+; GFX11-NEXT: v_add_f32_e64 v33, s0, 1.0
+; GFX11-NEXT: v_add_f32_e64 v24, s21, 1.0
+; GFX11-NEXT: v_add_f32_e64 v23, s20, 1.0
+; GFX11-NEXT: v_add_f32_e64 v10, s13, 1.0
+; GFX11-NEXT: v_add_f32_e64 v2, s5, 1.0
+; GFX11-NEXT: v_add_f32_e64 v1, s4, 1.0
+; GFX11-NEXT: v_add_f32_e64 v9, s12, 1.0
+; GFX11-NEXT: v_lshrrev_b64 v[49:50], 24, v[13:14]
+; GFX11-NEXT: v_lshrrev_b64 v[50:51], 24, v[15:16]
+; GFX11-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
+; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
+; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[25:26]
+; GFX11-NEXT: v_lshrrev_b64 v[36:37], 24, v[5:6]
+; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[19:20]
+; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[27:28]
+; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[7:8]
+; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[21:22]
+; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[31:32]
+; GFX11-NEXT: v_lshrrev_b64 v[29:30], 24, v[1:2]
+; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[9:10]
+; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[23:24]
+; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[33:34]
+; GFX11-NEXT: v_lshrrev_b32_e32 v70, 24, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v69, 16, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v39, 8, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v30, 8, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v83, 24, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v81, 16, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v55, 8, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v82, 8, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v86, 24, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v80, 8, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v98, 16, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v96, 8, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v100, 24, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v99, 16, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v101, 16, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v97, 8, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v112, 24, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v102, 8, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v116, 16, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v114, 8, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v118, 24, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v117, 16, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v113, 8, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v119, 16, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v115, 8, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v129, 16, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v128, 8, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v134, 16, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v144, 24, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v135, 16, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v131, 8, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v15
+; GFX11-NEXT: v_lshrrev_b32_e32 v133, 8, v15
+; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v147, 16, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v146, 8, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v151, 16, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v150, 8, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v163, 16, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v160, 8, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v166, 24, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v165, 16, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v164, 8, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v177, 16, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v180, 24, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v176, 8, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v178, 8, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v41, 24, v26
+; GFX11-NEXT: v_lshrrev_b32_e32 v40, 16, v26
+; GFX11-NEXT: v_lshrrev_b32_e32 v182, 8, v26
+; GFX11-NEXT: v_lshrrev_b32_e32 v43, 16, v25
+; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v25
+; GFX11-NEXT: v_lshrrev_b32_e32 v45, 24, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v44, 16, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v42, 8, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v47, 16, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v46, 8, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v59, 24, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v58, 16, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v56, 8, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v60, 16, v31
+; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v31
+; GFX11-NEXT: v_lshrrev_b32_e32 v62, 24, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v61, 16, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v63, 8, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v73, 16, v33
+; GFX11-NEXT: v_lshrrev_b32_e32 v72, 8, v33
+; GFX11-NEXT: s_branch .LBB37_6
+; GFX11-NEXT: .LBB37_5:
; GFX11-NEXT: v_dual_mov_b32 v33, s0 :: v_dual_mov_b32 v34, s1
; GFX11-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-NEXT: v_dual_mov_b32 v31, s2 :: v_dual_mov_b32 v32, s3
@@ -47233,7 +47590,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v116, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 17
-; GFX11-NEXT: v_dual_mov_b32 v147, s34 :: v_dual_mov_b32 v148, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v147, s34 :: v_dual_mov_b32 v148, s31
; GFX11-NEXT: v_dual_mov_b32 v29, s72 :: v_dual_mov_b32 v48, s90
; GFX11-NEXT: v_mov_b32_e32 v102, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 18
@@ -47243,7 +47600,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_mov_b32_e32 v103, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 19
; GFX11-NEXT: v_dual_mov_b32 v49, s92 :: v_dual_mov_b32 v54, s58
-; GFX11-NEXT: v_dual_mov_b32 v51, s30 :: v_dual_mov_b32 v64, s56
+; GFX11-NEXT: v_dual_mov_b32 v51, vcc_lo :: v_dual_mov_b32 v64, s56
; GFX11-NEXT: v_mov_b32_e32 v112, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 20
; GFX11-NEXT: v_dual_mov_b32 v53, s60 :: v_dual_mov_b32 v66, s44
@@ -47299,7 +47656,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_mov_b32_e32 v69, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v70, s0
-; GFX11-NEXT: .LBB37_5: ; %end
+; GFX11-NEXT: .LBB37_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_perm_b32 v67, v73, v67, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v66, v60, v66, 0xc0c0004
@@ -54023,8 +54380,10 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_cbranch_vccnz .LBB39_5
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
@@ -55021,8 +55380,10 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; VI-NEXT: s_cbranch_vccnz .LBB39_5
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
@@ -55945,8 +56306,10 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB39_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
@@ -56441,7 +56804,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -56587,9 +56950,17 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB39_3
-; GFX11-TRUE16-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB39_3
+; GFX11-TRUE16-NEXT: .LBB39_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB39_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
@@ -56880,15 +57251,12 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-TRUE16-NEXT: .LBB39_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB39_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB39_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB39_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v32f32_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -57015,7 +57383,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -57161,9 +57529,17 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB39_3
-; GFX11-FAKE16-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB39_3
+; GFX11-FAKE16-NEXT: .LBB39_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB39_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
@@ -57454,15 +57830,12 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-FAKE16-NEXT: .LBB39_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB39_5: ; %end
; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB39_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB39_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -58342,16 +58715,14 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xffff0000
-; SI-NEXT: v_writelane_b32 v62, s4, 3
-; SI-NEXT: s_lshl_b32 s4, s6, 16
-; SI-NEXT: v_writelane_b32 v62, s4, 2
-; SI-NEXT: s_and_b32 s4, s7, 0xffff0000
; SI-NEXT: v_writelane_b32 v62, s4, 1
-; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: s_lshl_b32 s4, s6, 16
; SI-NEXT: v_writelane_b32 v62, s4, 0
+; SI-NEXT: s_and_b32 s58, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s59, s7, 16
; SI-NEXT: s_and_b32 s60, s8, 0xffff0000
; SI-NEXT: s_lshl_b32 s61, s8, 16
; SI-NEXT: s_and_b32 s62, s9, 0xffff0000
@@ -58372,164 +58743,51 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: s_lshl_b32 s93, s40, 16
; SI-NEXT: s_and_b32 s94, s41, 0xffff0000
; SI-NEXT: s_lshl_b32 s95, s41, 16
-; SI-NEXT: s_and_b32 s30, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s31, s42, 16
-; SI-NEXT: s_and_b32 s34, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s35, s43, 16
-; SI-NEXT: s_and_b32 s36, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s37, s44, 16
-; SI-NEXT: s_and_b32 s38, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s39, s45, 16
-; SI-NEXT: s_and_b32 s48, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s49, s46, 16
-; SI-NEXT: s_and_b32 s50, s47, 0xffff0000
-; SI-NEXT: s_lshl_b32 s51, s47, 16
-; SI-NEXT: s_and_b32 s52, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s53, s29, 16
-; SI-NEXT: s_and_b32 s54, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s55, s28, 16
-; SI-NEXT: s_and_b32 s64, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s27, 16
-; SI-NEXT: s_and_b32 s66, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s26, 16
-; SI-NEXT: s_and_b32 s68, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s69, s25, 16
-; SI-NEXT: s_and_b32 s70, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s24, 16
-; SI-NEXT: s_and_b32 s80, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s23, 16
-; SI-NEXT: s_and_b32 s82, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s22, 16
-; SI-NEXT: s_and_b32 s84, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s21, 16
-; SI-NEXT: s_and_b32 s86, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s20, 16
-; SI-NEXT: s_and_b32 s96, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s19, 16
-; SI-NEXT: s_and_b32 s98, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s99, s18, 16
-; SI-NEXT: s_and_b32 s56, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s57, s17, 16
-; SI-NEXT: s_and_b32 s58, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s59, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v29, s6, 1.0
-; SI-NEXT: v_add_f32_e64 v28, s7, 1.0
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_add_f32_e64 v27, s8, 1.0
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_add_f32_e64 v0, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v5, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s47, 1.0
-; SI-NEXT: v_add_f32_e64 v13, s46, 1.0
-; SI-NEXT: v_add_f32_e64 v14, s45, 1.0
-; SI-NEXT: v_add_f32_e64 v15, s44, 1.0
-; SI-NEXT: v_add_f32_e64 v26, s9, 1.0
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e64 v2, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v3, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v9, s27, 1.0
-; SI-NEXT: v_add_f32_e64 v10, s28, 1.0
-; SI-NEXT: v_add_f32_e64 v11, s29, 1.0
-; SI-NEXT: v_add_f32_e64 v22, s13, 1.0
-; SI-NEXT: v_add_f32_e64 v23, s12, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s11, 1.0
-; SI-NEXT: v_add_f32_e64 v25, s10, 1.0
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v15
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v13
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v13
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v12
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v12
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v5
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; SI-NEXT: v_add_f32_e64 v0, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s24, 1.0
-; SI-NEXT: v_add_f32_e64 v7, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v8, s26, 1.0
-; SI-NEXT: v_add_f32_e64 v16, s43, 1.0
-; SI-NEXT: v_add_f32_e64 v17, s42, 1.0
-; SI-NEXT: v_add_f32_e64 v18, s41, 1.0
-; SI-NEXT: v_add_f32_e64 v19, s40, 1.0
-; SI-NEXT: v_add_f32_e64 v20, s15, 1.0
-; SI-NEXT: v_add_f32_e64 v21, s14, 1.0
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v25
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v24
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v23
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v22
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v11
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v10
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v9
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v2
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v21
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v20
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v19
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v18
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v17
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v16
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v8
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v7
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v6
-; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
+; SI-NEXT: s_and_b32 vcc_lo, s42, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_hi, s42, 16
+; SI-NEXT: s_and_b32 s30, s43, 0xffff0000
+; SI-NEXT: s_lshl_b32 s31, s43, 16
+; SI-NEXT: s_and_b32 s34, s44, 0xffff0000
+; SI-NEXT: s_lshl_b32 s35, s44, 16
+; SI-NEXT: s_and_b32 s36, s45, 0xffff0000
+; SI-NEXT: s_lshl_b32 s37, s45, 16
+; SI-NEXT: s_and_b32 s38, s46, 0xffff0000
+; SI-NEXT: s_lshl_b32 s39, s46, 16
+; SI-NEXT: s_and_b32 s48, s47, 0xffff0000
+; SI-NEXT: s_lshl_b32 s49, s47, 16
+; SI-NEXT: s_and_b32 s50, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s51, s29, 16
+; SI-NEXT: s_and_b32 s52, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 s53, s28, 16
+; SI-NEXT: s_and_b32 s54, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 s55, s27, 16
+; SI-NEXT: s_and_b32 s64, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 s65, s26, 16
+; SI-NEXT: s_and_b32 s66, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s67, s25, 16
+; SI-NEXT: s_and_b32 s68, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s69, s24, 16
+; SI-NEXT: s_and_b32 s70, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s71, s23, 16
+; SI-NEXT: s_and_b32 s80, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s81, s22, 16
+; SI-NEXT: s_and_b32 s82, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s83, s21, 16
+; SI-NEXT: s_and_b32 s84, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s85, s20, 16
+; SI-NEXT: s_and_b32 s86, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s87, s19, 16
+; SI-NEXT: s_and_b32 s96, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s97, s18, 16
+; SI-NEXT: s_and_b32 s98, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s99, s17, 16
+; SI-NEXT: s_and_b32 s56, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s57, s16, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr99
@@ -58568,6 +58826,8 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: ; implicit-def: $sgpr34
; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr93
@@ -58588,14 +58848,129 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f32_e64 v29, s6, 1.0
+; SI-NEXT: v_add_f32_e64 v28, s7, 1.0
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v29
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: v_add_f32_e64 v27, s8, 1.0
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_add_f32_e64 v0, s18, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s22, 1.0
+; SI-NEXT: v_add_f32_e64 v5, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s47, 1.0
+; SI-NEXT: v_add_f32_e64 v13, s46, 1.0
+; SI-NEXT: v_add_f32_e64 v14, s45, 1.0
+; SI-NEXT: v_add_f32_e64 v15, s44, 1.0
+; SI-NEXT: v_add_f32_e64 v26, s9, 1.0
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e64 v2, s20, 1.0
+; SI-NEXT: v_add_f32_e64 v3, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v9, s27, 1.0
+; SI-NEXT: v_add_f32_e64 v10, s28, 1.0
+; SI-NEXT: v_add_f32_e64 v11, s29, 1.0
+; SI-NEXT: v_add_f32_e64 v22, s13, 1.0
+; SI-NEXT: v_add_f32_e64 v23, s12, 1.0
+; SI-NEXT: v_add_f32_e64 v24, s11, 1.0
+; SI-NEXT: v_add_f32_e64 v25, s10, 1.0
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v15
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v13
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v13
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v12
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v12
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v5
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v5
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; SI-NEXT: v_add_f32_e64 v0, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v1, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v6, s24, 1.0
+; SI-NEXT: v_add_f32_e64 v7, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v8, s26, 1.0
+; SI-NEXT: v_add_f32_e64 v16, s43, 1.0
+; SI-NEXT: v_add_f32_e64 v17, s42, 1.0
+; SI-NEXT: v_add_f32_e64 v18, s41, 1.0
+; SI-NEXT: v_add_f32_e64 v19, s40, 1.0
+; SI-NEXT: v_add_f32_e64 v20, s15, 1.0
+; SI-NEXT: v_add_f32_e64 v21, s14, 1.0
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v25
+; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v24
+; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v23
+; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v22
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v22
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v11
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v10
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v9
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v2
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
+; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v21
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v21
+; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v20
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v20
+; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v19
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v19
+; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v18
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v18
+; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v17
+; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v16
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v8
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v7
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v6
+; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v7, s63
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -58603,62 +58978,60 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, s60
-; SI-NEXT: v_readlane_b32 s4, v62, 0
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 1
+; SI-NEXT: v_mov_b32_e32 v7, s59
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 2
+; SI-NEXT: v_mov_b32_e32 v7, s58
+; SI-NEXT: v_readlane_b32 s4, v62, 0
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 3
+; SI-NEXT: v_readlane_b32 s4, v62, 1
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_mov_b32_e32 v0, s59
-; SI-NEXT: v_mov_b32_e32 v1, s58
-; SI-NEXT: v_mov_b32_e32 v2, s57
-; SI-NEXT: v_mov_b32_e32 v3, s56
-; SI-NEXT: v_mov_b32_e32 v4, s99
-; SI-NEXT: v_mov_b32_e32 v5, s98
-; SI-NEXT: v_mov_b32_e32 v6, s97
-; SI-NEXT: v_mov_b32_e32 v39, s96
-; SI-NEXT: v_mov_b32_e32 v38, s87
-; SI-NEXT: v_mov_b32_e32 v9, s86
-; SI-NEXT: v_mov_b32_e32 v10, s85
-; SI-NEXT: v_mov_b32_e32 v11, s84
-; SI-NEXT: v_mov_b32_e32 v12, s83
-; SI-NEXT: v_mov_b32_e32 v13, s82
-; SI-NEXT: v_mov_b32_e32 v14, s81
-; SI-NEXT: v_mov_b32_e32 v15, s80
-; SI-NEXT: v_mov_b32_e32 v48, s71
-; SI-NEXT: v_mov_b32_e32 v17, s70
-; SI-NEXT: v_mov_b32_e32 v18, s69
-; SI-NEXT: v_mov_b32_e32 v19, s68
-; SI-NEXT: v_mov_b32_e32 v20, s67
-; SI-NEXT: v_mov_b32_e32 v21, s66
-; SI-NEXT: v_mov_b32_e32 v22, s65
-; SI-NEXT: v_mov_b32_e32 v23, s64
-; SI-NEXT: v_mov_b32_e32 v24, s55
-; SI-NEXT: v_mov_b32_e32 v25, s54
-; SI-NEXT: v_mov_b32_e32 v26, s53
-; SI-NEXT: v_mov_b32_e32 v27, s52
-; SI-NEXT: v_mov_b32_e32 v28, s51
-; SI-NEXT: v_mov_b32_e32 v29, s50
-; SI-NEXT: v_mov_b32_e32 v30, s49
-; SI-NEXT: v_mov_b32_e32 v31, s48
-; SI-NEXT: v_mov_b32_e32 v32, s39
-; SI-NEXT: v_mov_b32_e32 v33, s38
-; SI-NEXT: v_mov_b32_e32 v36, s37
-; SI-NEXT: v_mov_b32_e32 v37, s36
-; SI-NEXT: v_mov_b32_e32 v60, s35
-; SI-NEXT: v_mov_b32_e32 v61, s34
-; SI-NEXT: v_mov_b32_e32 v58, s31
-; SI-NEXT: v_mov_b32_e32 v59, s30
+; SI-NEXT: v_mov_b32_e32 v0, s57
+; SI-NEXT: v_mov_b32_e32 v1, s56
+; SI-NEXT: v_mov_b32_e32 v2, s99
+; SI-NEXT: v_mov_b32_e32 v3, s98
+; SI-NEXT: v_mov_b32_e32 v4, s97
+; SI-NEXT: v_mov_b32_e32 v5, s96
+; SI-NEXT: v_mov_b32_e32 v6, s87
+; SI-NEXT: v_mov_b32_e32 v39, s86
+; SI-NEXT: v_mov_b32_e32 v38, s85
+; SI-NEXT: v_mov_b32_e32 v9, s84
+; SI-NEXT: v_mov_b32_e32 v10, s83
+; SI-NEXT: v_mov_b32_e32 v11, s82
+; SI-NEXT: v_mov_b32_e32 v12, s81
+; SI-NEXT: v_mov_b32_e32 v13, s80
+; SI-NEXT: v_mov_b32_e32 v14, s71
+; SI-NEXT: v_mov_b32_e32 v15, s70
+; SI-NEXT: v_mov_b32_e32 v48, s69
+; SI-NEXT: v_mov_b32_e32 v17, s68
+; SI-NEXT: v_mov_b32_e32 v18, s67
+; SI-NEXT: v_mov_b32_e32 v19, s66
+; SI-NEXT: v_mov_b32_e32 v20, s65
+; SI-NEXT: v_mov_b32_e32 v21, s64
+; SI-NEXT: v_mov_b32_e32 v22, s55
+; SI-NEXT: v_mov_b32_e32 v23, s54
+; SI-NEXT: v_mov_b32_e32 v24, s53
+; SI-NEXT: v_mov_b32_e32 v25, s52
+; SI-NEXT: v_mov_b32_e32 v26, s51
+; SI-NEXT: v_mov_b32_e32 v27, s50
+; SI-NEXT: v_mov_b32_e32 v28, s49
+; SI-NEXT: v_mov_b32_e32 v29, s48
+; SI-NEXT: v_mov_b32_e32 v30, s39
+; SI-NEXT: v_mov_b32_e32 v31, s38
+; SI-NEXT: v_mov_b32_e32 v32, s37
+; SI-NEXT: v_mov_b32_e32 v33, s36
+; SI-NEXT: v_mov_b32_e32 v36, s35
+; SI-NEXT: v_mov_b32_e32 v37, s34
+; SI-NEXT: v_mov_b32_e32 v60, s31
+; SI-NEXT: v_mov_b32_e32 v61, s30
+; SI-NEXT: v_mov_b32_e32 v58, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v59, vcc_lo
; SI-NEXT: v_mov_b32_e32 v56, s95
; SI-NEXT: v_mov_b32_e32 v57, s94
; SI-NEXT: v_mov_b32_e32 v46, s93
@@ -58677,7 +59050,7 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; SI-NEXT: v_mov_b32_e32 v51, s72
; SI-NEXT: v_mov_b32_e32 v49, s62
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_mul_f32_e32 v34, 1.0, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
@@ -58936,10 +59309,18 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -58972,10 +59353,8 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; VI-NEXT: v_add_f32_e64 v18, s38, 1.0
; VI-NEXT: v_add_f32_e64 v17, s37, 1.0
; VI-NEXT: v_add_f32_e64 v16, s36, 1.0
-; VI-NEXT: s_branch .LBB41_5
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: s_branch .LBB41_6
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -59008,7 +59387,7 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; VI-NEXT: v_mov_b32_e32 v29, s49
; VI-NEXT: v_mov_b32_e32 v30, s50
; VI-NEXT: v_mov_b32_e32 v31, s51
-; VI-NEXT: .LBB41_5: ; %end
+; VI-NEXT: .LBB41_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -59061,10 +59440,18 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -59097,10 +59484,8 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; GFX9-NEXT: v_add_f32_e64 v18, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s36, 1.0
-; GFX9-NEXT: s_branch .LBB41_5
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: s_branch .LBB41_6
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s17
@@ -59133,7 +59518,7 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; GFX9-NEXT: v_mov_b32_e32 v29, s49
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
-; GFX9-NEXT: .LBB41_5: ; %end
+; GFX9-NEXT: .LBB41_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -59187,11 +59572,16 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -59225,8 +59615,6 @@ define inreg <64 x bfloat> @bitcast_v32f32_to_v64bf16_scalar(<32 x float> inreg
; GFX11-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s36, 1.0
; GFX11-NEXT: s_branch .LBB41_5
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -62774,7 +63162,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_mov_b32_e32 v33, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
@@ -62791,7 +63181,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v40, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_cbranch_vccnz .LBB43_5
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -63161,10 +63551,18 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s50, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -63757,10 +64155,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB43_5
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: s_branch .LBB43_6
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -63793,7 +64189,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: .LBB43_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v35, 15
; VI-NEXT: v_readlane_b32 s66, v35, 14
; VI-NEXT: v_readlane_b32 s65, v35, 13
@@ -63872,10 +64268,18 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -64486,10 +64890,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -64522,7 +64924,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v36, 15
; GFX9-NEXT: v_readlane_b32 s66, v36, 14
; GFX9-NEXT: v_readlane_b32 s65, v36, 13
@@ -64602,11 +65004,16 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s37, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-TRUE16-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB43_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s51, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s51, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -65229,8 +65636,6 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v34
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v33.l
; GFX11-TRUE16-NEXT: s_branch .LBB43_5
-; GFX11-TRUE16-NEXT: .LBB43_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB43_2
; GFX11-TRUE16-NEXT: .LBB43_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -65328,11 +65733,16 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX11-FAKE16-NEXT: s_mov_b32 s37, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-FAKE16-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB43_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s51, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s51, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -65986,8 +66396,6 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v32, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v36, 16, v33
; GFX11-FAKE16-NEXT: s_branch .LBB43_5
-; GFX11-FAKE16-NEXT: .LBB43_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB43_2
; GFX11-FAKE16-NEXT: .LBB43_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -66555,7 +66963,7 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s69, s5, 16
; SI-NEXT: s_lshr_b32 s68, s7, 16
@@ -66573,9 +66981,9 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b32 s48, s21, 16
; SI-NEXT: s_lshr_b32 s39, s19, 16
; SI-NEXT: s_lshr_b32 s38, s17, 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[8:9], 16
+; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[14:15], 16
@@ -66589,8 +66997,48 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v31, s5, 1.0
; SI-NEXT: v_add_f32_e64 v30, s4, 1.0
; SI-NEXT: v_lshr_b64 v[32:33], v[30:31], 16
@@ -66658,43 +67106,9 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: v_mov_b32_e32 v33, s30
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
+; SI-NEXT: v_mov_b32_e32 v33, vcc_lo
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v0, s16
@@ -66747,9 +67161,9 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v41, s68
; SI-NEXT: v_mov_b32_e32 v55, s69
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, s34
+; SI-NEXT: v_mov_b32_e32 v33, s30
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v34, s36
+; SI-NEXT: v_mov_b32_e32 v34, s34
; SI-NEXT: v_mov_b32_e32 v35, s46
; SI-NEXT: v_mov_b32_e32 v36, s56
; SI-NEXT: v_mov_b32_e32 v37, s58
@@ -66763,7 +67177,7 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v54, s90
; SI-NEXT: v_mov_b32_e32 v39, s92
; SI-NEXT: v_mov_b32_e32 v40, s94
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v1, v1, v32
@@ -66942,10 +67356,18 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -66978,10 +67400,8 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v18, s38, 1.0
; VI-NEXT: v_add_f32_e64 v17, s37, 1.0
; VI-NEXT: v_add_f32_e64 v16, s36, 1.0
-; VI-NEXT: s_branch .LBB45_5
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: s_branch .LBB45_6
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -67014,7 +67434,7 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s49
; VI-NEXT: v_mov_b32_e32 v30, s50
; VI-NEXT: v_mov_b32_e32 v31, s51
-; VI-NEXT: .LBB45_5: ; %end
+; VI-NEXT: .LBB45_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -67067,10 +67487,18 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -67103,10 +67531,8 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v18, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s36, 1.0
-; GFX9-NEXT: s_branch .LBB45_5
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: s_branch .LBB45_6
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s17
@@ -67139,7 +67565,7 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s49
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
-; GFX9-NEXT: .LBB45_5: ; %end
+; GFX9-NEXT: .LBB45_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -67193,11 +67619,16 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -67231,8 +67662,6 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s36, 1.0
; GFX11-NEXT: s_branch .LBB45_5
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -68286,10 +68715,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_writelane_b32 v34, s30, 34
; SI-NEXT: v_writelane_b32 v34, s31, 35
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 vcc_hi, s8, 16
; SI-NEXT: v_readfirstlane_b32 s10, v15
; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: v_readfirstlane_b32 s14, v13
@@ -68301,29 +68727,29 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s90, v7
; SI-NEXT: v_readfirstlane_b32 s92, v6
; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s34, v3
-; SI-NEXT: v_readfirstlane_b32 s68, v2
-; SI-NEXT: v_readfirstlane_b32 s80, v1
-; SI-NEXT: v_readfirstlane_b32 s83, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v35, vcc_lo, 0
-; SI-NEXT: s_lshr_b32 s69, s29, 16
-; SI-NEXT: s_lshr_b32 s71, s28, 16
-; SI-NEXT: s_lshr_b32 s82, s27, 16
-; SI-NEXT: s_lshr_b32 s85, s26, 16
-; SI-NEXT: s_lshr_b32 s86, s25, 16
-; SI-NEXT: s_lshr_b32 s87, s24, 16
-; SI-NEXT: s_lshr_b32 s96, s23, 16
-; SI-NEXT: s_lshr_b32 s97, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s7, s19, 16
-; SI-NEXT: s_lshr_b32 s9, s18, 16
-; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: s_lshr_b32 s13, s16, 16
-; SI-NEXT: s_lshr_b32 s65, s10, 16
-; SI-NEXT: s_lshr_b32 s64, s12, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s30, v3
+; SI-NEXT: v_readfirstlane_b32 s34, v2
+; SI-NEXT: v_readfirstlane_b32 s70, v1
+; SI-NEXT: v_readfirstlane_b32 s81, v0
+; SI-NEXT: s_lshr_b32 s35, s29, 16
+; SI-NEXT: s_lshr_b32 s69, s28, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 16
+; SI-NEXT: s_lshr_b32 s83, s26, 16
+; SI-NEXT: s_lshr_b32 s84, s25, 16
+; SI-NEXT: s_lshr_b32 s85, s24, 16
+; SI-NEXT: s_lshr_b32 s86, s23, 16
+; SI-NEXT: s_lshr_b32 s87, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s7, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s6, 16
+; SI-NEXT: s_lshr_b32 s66, s8, 16
+; SI-NEXT: s_lshr_b32 s11, s10, 16
+; SI-NEXT: s_lshr_b32 s13, s12, 16
; SI-NEXT: s_lshr_b32 s15, s14, 16
; SI-NEXT: s_lshr_b32 s73, s72, 16
; SI-NEXT: s_lshr_b32 s75, s74, 16
@@ -68333,74 +68759,75 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s31, s30, 16
-; SI-NEXT: s_lshr_b32 s35, s34, 16
-; SI-NEXT: s_lshr_b32 s70, s68, 16
-; SI-NEXT: s_lshr_b32 s81, s80, 16
-; SI-NEXT: s_lshr_b32 s84, s83, 16
+; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 s71, s70, 16
+; SI-NEXT: s_lshr_b32 s82, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v35, vcc_hi, 1
+; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v35, s65, 2
-; SI-NEXT: v_writelane_b32 v35, s64, 3
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v35, s67, 0
+; SI-NEXT: v_writelane_b32 v35, s66, 1
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s11, 16
+; SI-NEXT: s_lshl_b32 s41, s7, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
-; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s30, 0xffff
; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -68430,22 +68857,31 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s15, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s7
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -68457,7 +68893,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s9
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s99
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -68465,11 +68901,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s98
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s99
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s97
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -68480,7 +68916,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s96
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -68489,11 +68925,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s86
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -68504,7 +68940,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s85
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -68513,11 +68949,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s84
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s83
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -68528,7 +68964,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s80
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -68537,11 +68973,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: v_cvt_f32_f16_e32 v13, s28
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s35
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
@@ -68552,8 +68988,8 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v15, s29
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s84
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s81
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -68561,11 +68997,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s71
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s80
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s68
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
@@ -68573,11 +69009,11 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s34
; SI-NEXT: v_or_b32_e32 v15, v16, v15
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
@@ -68585,10 +69021,10 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
; SI-NEXT: v_or_b32_e32 v16, v18, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v18, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v19, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v19, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v20, s95
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
@@ -68659,10 +69095,9 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v25, v27, v25
; SI-NEXT: v_cvt_f32_f16_e32 v27, s15
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_readlane_b32 s4, v35, 3
; SI-NEXT: v_or_b32_e32 v26, v28, v26
; SI-NEXT: v_cvt_f32_f16_e32 v28, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s13
; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v28
@@ -68670,11 +69105,10 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_readlane_b32 s4, v35, 2
; SI-NEXT: v_cvt_f32_f16_e32 v30, s12
; SI-NEXT: v_or_b32_e32 v27, v28, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s11
; SI-NEXT: v_cvt_f32_f16_e32 v31, s10
; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
@@ -68703,11 +69137,8 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v30, v31, v30
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v33, v31
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -68740,7 +69171,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v34, 34
; SI-NEXT: v_readlane_b32 s31, v34, 35
; SI-NEXT: v_readlane_b32 s99, v34, 33
@@ -68840,10 +69271,18 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s51, 16
; VI-NEXT: v_mov_b32_e32 v16, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -69005,10 +69444,8 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s52, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -69041,7 +69478,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
+; VI-NEXT: .LBB47_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v33, 15
; VI-NEXT: v_readlane_b32 s66, v33, 14
; VI-NEXT: v_readlane_b32 s65, v33, 13
@@ -69120,10 +69557,18 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s51, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s50, v16 op_sel_hi:[1,0]
@@ -69157,10 +69602,8 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v18, s54, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -69193,7 +69636,7 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -69273,11 +69716,16 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s51 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s50 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s49 op_sel_hi:[0,1]
@@ -69311,8 +69759,6 @@ define inreg <32 x float> @bitcast_v64f16_to_v32f32_scalar(<64 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s53 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s52 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB47_5
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -69880,7 +70326,7 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s69, s5, 16
; SI-NEXT: s_lshr_b32 s68, s7, 16
@@ -69898,9 +70344,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b32 s48, s21, 16
; SI-NEXT: s_lshr_b32 s39, s19, 16
; SI-NEXT: s_lshr_b32 s38, s17, 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[8:9], 16
+; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[12:13], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[14:15], 16
@@ -69914,8 +70360,48 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v31, s5, 1.0
; SI-NEXT: v_add_f32_e64 v30, s4, 1.0
; SI-NEXT: v_lshr_b64 v[32:33], v[30:31], 16
@@ -69983,43 +70469,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
-; SI-NEXT: v_mov_b32_e32 v33, s30
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
+; SI-NEXT: v_mov_b32_e32 v33, vcc_lo
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v0, s16
@@ -70072,9 +70524,9 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v41, s68
; SI-NEXT: v_mov_b32_e32 v55, s69
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, s34
+; SI-NEXT: v_mov_b32_e32 v33, s30
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v34, s36
+; SI-NEXT: v_mov_b32_e32 v34, s34
; SI-NEXT: v_mov_b32_e32 v35, s46
; SI-NEXT: v_mov_b32_e32 v36, s56
; SI-NEXT: v_mov_b32_e32 v37, s58
@@ -70088,7 +70540,7 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v54, s90
; SI-NEXT: v_mov_b32_e32 v39, s92
; SI-NEXT: v_mov_b32_e32 v40, s94
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v1, v1, v32
@@ -70267,10 +70719,18 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -70303,10 +70763,8 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v18, s38, 1.0
; VI-NEXT: v_add_f32_e64 v17, s37, 1.0
; VI-NEXT: v_add_f32_e64 v16, s36, 1.0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -70339,7 +70797,7 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s49
; VI-NEXT: v_mov_b32_e32 v30, s50
; VI-NEXT: v_mov_b32_e32 v31, s51
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -70392,10 +70850,18 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -70428,10 +70894,8 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v18, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s36, 1.0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s17
@@ -70464,7 +70928,7 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s49
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -70518,11 +70982,16 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB49_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -70556,8 +71025,6 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v17, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s36, 1.0
; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: s_branch .LBB49_2
; GFX11-NEXT: .LBB49_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -71440,13 +71907,10 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s99, 33
; SI-NEXT: v_writelane_b32 v32, s30, 34
; SI-NEXT: v_writelane_b32 v32, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s9, v16
-; SI-NEXT: s_lshr_b32 s14, s9, 16
-; SI-NEXT: v_readfirstlane_b32 s13, v14
-; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
; SI-NEXT: v_readfirstlane_b32 s11, v15
-; SI-NEXT: s_lshr_b32 s72, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s13, v14
; SI-NEXT: v_readfirstlane_b32 s15, v13
; SI-NEXT: v_readfirstlane_b32 s73, v12
; SI-NEXT: v_readfirstlane_b32 s75, v11
@@ -71455,30 +71919,30 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s89, v8
; SI-NEXT: v_readfirstlane_b32 s91, v7
; SI-NEXT: v_readfirstlane_b32 s94, v6
-; SI-NEXT: v_readfirstlane_b32 s31, v5
-; SI-NEXT: v_readfirstlane_b32 s68, v4
-; SI-NEXT: v_readfirstlane_b32 s80, v3
-; SI-NEXT: v_readfirstlane_b32 s83, v2
-; SI-NEXT: v_readfirstlane_b32 s86, v1
-; SI-NEXT: v_readfirstlane_b32 s97, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; SI-NEXT: v_readfirstlane_b32 s34, v4
+; SI-NEXT: v_readfirstlane_b32 s70, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s84, v1
+; SI-NEXT: v_readfirstlane_b32 s87, v0
; SI-NEXT: s_lshr_b32 s92, s29, 16
; SI-NEXT: s_lshr_b32 s95, s28, 16
-; SI-NEXT: s_lshr_b32 s34, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s71, s25, 16
-; SI-NEXT: s_lshr_b32 s81, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s87, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s6, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s10, s17, 16
-; SI-NEXT: s_lshr_b32 s12, s16, 16
-; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s11, 16
+; SI-NEXT: s_lshr_b32 s30, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s69, s25, 16
+; SI-NEXT: s_lshr_b32 s71, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s85, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s10, s9, 16
+; SI-NEXT: s_lshr_b32 s12, s11, 16
+; SI-NEXT: s_lshr_b32 s14, s13, 16
; SI-NEXT: s_lshr_b32 s63, s15, 16
; SI-NEXT: s_lshr_b32 s62, s73, 16
; SI-NEXT: s_lshr_b32 s61, s75, 16
@@ -71487,54 +71951,55 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s68, 16
-; SI-NEXT: s_lshr_b32 s70, s80, 16
-; SI-NEXT: s_lshr_b32 s82, s83, 16
-; SI-NEXT: s_lshr_b32 s85, s86, 16
-; SI-NEXT: s_lshr_b32 s96, s97, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s70, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v33, s72, 1
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v33, s63, 2
-; SI-NEXT: v_writelane_b32 v33, s62, 3
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_writelane_b32 v33, s63, 1
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s10, 16
+; SI-NEXT: s_lshl_b32 s41, s6, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -71542,23 +72007,23 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s54, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s55, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -71574,92 +72039,109 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_or_b32 s59, s4, s5
; SI-NEXT: s_and_b32 s4, s77, 0xffff
; SI-NEXT: s_lshl_b32 s5, s60, 16
-; SI-NEXT: s_mov_b32 s76, s60
+; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: s_or_b32 s60, s4, s5
; SI-NEXT: s_and_b32 s4, s75, 0xffff
; SI-NEXT: s_lshl_b32 s5, s61, 16
-; SI-NEXT: s_mov_b32 s74, s61
+; SI-NEXT: s_mov_b32 s76, s61
; SI-NEXT: s_or_b32 s61, s4, s5
; SI-NEXT: s_and_b32 s4, s73, 0xffff
; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_mov_b32 s72, s62
; SI-NEXT: s_or_b32 s62, s4, s5
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b32 s74, s60
+; SI-NEXT: s_mov_b32 s76, s61
+; SI-NEXT: s_mov_b32 s72, s62
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_add_i32 s37, s5, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -71671,35 +72153,35 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s97, s97, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
+; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s34, s34, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s54, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s55, s4, 0x30000
@@ -71724,27 +72206,26 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s74, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s75, s75, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s73, s73, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s72, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s15, s15, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 2
+; SI-NEXT: v_readlane_b32 s5, v33, 1
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 1
+; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s63, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s13, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
@@ -71752,21 +72233,20 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_add_i32 s64, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s65, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s66, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s67, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_readlane_b32 s30, v32, 34
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
@@ -71841,13 +72321,6 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: s_mov_b32 s76, s60
-; SI-NEXT: s_mov_b32 s74, s61
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v64i16_to_v32f32_scalar:
; VI: ; %bb.0:
@@ -71872,10 +72345,18 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s46, 3
; VI-NEXT: s_and_b32 s4, s46, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -72036,7 +72517,7 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s45, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -72070,8 +72551,6 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v64i16_to_v32f32_scalar:
; GFX9: ; %bb.0:
@@ -72129,10 +72608,18 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -72165,10 +72652,8 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -72201,7 +72686,7 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -72281,11 +72766,16 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -72319,8 +72809,6 @@ define inreg <32 x float> @bitcast_v64i16_to_v32f32_scalar(<64 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB51_5
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -72628,10 +73116,18 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -72664,7 +73160,7 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -72698,8 +73194,6 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s7
; SI-NEXT: v_mov_b32_e32 v31, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v16i64_to_v16f64_scalar:
; VI: ; %bb.0:
@@ -72724,10 +73218,18 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -72760,7 +73262,7 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -72794,8 +73296,6 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v16i64_to_v16f64_scalar:
; GFX9: ; %bb.0:
@@ -72820,10 +73320,18 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -72856,7 +73364,7 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -72890,8 +73398,6 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v16i64_to_v16f64_scalar:
; GFX11: ; %bb.0:
@@ -72913,11 +73419,16 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -72950,7 +73461,7 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -72968,8 +73479,6 @@ define inreg <16 x double> @bitcast_v16i64_to_v16f64_scalar(<16 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -73200,10 +73709,18 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -73220,10 +73737,8 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; SI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -73256,7 +73771,7 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -73335,10 +73850,18 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -73355,10 +73878,8 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; VI-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -73391,7 +73912,7 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
+; VI-NEXT: .LBB55_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -73470,10 +73991,18 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -73490,10 +74019,8 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX9-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -73526,7 +74053,7 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -73606,11 +74133,16 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -73628,8 +74160,6 @@ define inreg <16 x i64> @bitcast_v16f64_to_v16i64_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: v_add_f64 v[30:31], s[66:67], 1.0
; GFX11-NEXT: s_branch .LBB55_5
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -77726,179 +78256,179 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s45, v2
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr21 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s46, s5, 24
+; SI-NEXT: s_lshr_b32 s46, s21, 24
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s46, 46
-; SI-NEXT: s_lshr_b32 s46, s5, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 47
-; SI-NEXT: s_lshr_b32 s46, s5, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 48
-; SI-NEXT: s_lshr_b32 s46, s7, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 49
-; SI-NEXT: s_lshr_b32 s46, s7, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 50
-; SI-NEXT: s_lshr_b32 s46, s7, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 51
-; SI-NEXT: s_lshr_b32 s46, s9, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 52
-; SI-NEXT: s_lshr_b32 s46, s9, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 53
-; SI-NEXT: s_lshr_b32 s46, s9, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 54
-; SI-NEXT: s_lshr_b32 s46, s11, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 55
-; SI-NEXT: s_lshr_b32 s46, s11, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 56
-; SI-NEXT: s_lshr_b32 s46, s11, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 57
-; SI-NEXT: s_lshr_b32 s46, s13, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 58
-; SI-NEXT: s_lshr_b32 s46, s13, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 59
-; SI-NEXT: s_lshr_b32 s46, s13, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 60
-; SI-NEXT: s_lshr_b32 s46, s15, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 61
-; SI-NEXT: s_lshr_b32 s46, s15, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 62
-; SI-NEXT: s_lshr_b32 s46, s15, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 63
-; SI-NEXT: s_lshr_b32 s46, s41, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 0
-; SI-NEXT: s_lshr_b32 s46, s41, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 1
-; SI-NEXT: s_lshr_b32 s46, s41, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 2
-; SI-NEXT: s_lshr_b32 s46, s43, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 3
-; SI-NEXT: s_lshr_b32 s46, s43, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 4
-; SI-NEXT: s_lshr_b32 s46, s43, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 5
-; SI-NEXT: s_lshr_b32 s46, s45, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 6
-; SI-NEXT: s_lshr_b32 s46, s45, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 7
-; SI-NEXT: s_lshr_b32 s46, s45, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 8
-; SI-NEXT: s_lshr_b32 s46, s29, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 9
-; SI-NEXT: s_lshr_b32 s46, s29, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 10
-; SI-NEXT: s_lshr_b32 s46, s29, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 11
-; SI-NEXT: s_lshr_b32 s46, s27, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 12
-; SI-NEXT: s_lshr_b32 s46, s27, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 13
-; SI-NEXT: s_lshr_b32 s46, s27, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 14
-; SI-NEXT: s_lshr_b32 s46, s25, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 15
-; SI-NEXT: s_lshr_b32 s46, s25, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 16
-; SI-NEXT: s_lshr_b32 s46, s25, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 17
+; SI-NEXT: v_writelane_b32 v22, s46, 10
+; SI-NEXT: s_lshr_b32 s46, s21, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 11
+; SI-NEXT: s_lshr_b32 s46, s19, 24
+; SI-NEXT: v_writelane_b32 v22, s46, 12
+; SI-NEXT: s_lshr_b32 s46, s19, 16
+; SI-NEXT: v_writelane_b32 v22, s46, 13
+; SI-NEXT: s_lshr_b32 s46, s19, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 14
+; SI-NEXT: s_lshr_b32 s46, s17, 24
+; SI-NEXT: v_writelane_b32 v22, s46, 15
+; SI-NEXT: s_lshr_b32 s46, s17, 16
+; SI-NEXT: v_writelane_b32 v22, s46, 16
+; SI-NEXT: s_lshr_b32 s46, s17, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 17
; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 44
-; SI-NEXT: v_writelane_b32 v22, s47, 45
+; SI-NEXT: v_writelane_b32 v22, s46, 6
+; SI-NEXT: v_writelane_b32 v22, s47, 7
+; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
+; SI-NEXT: v_writelane_b32 v22, s46, 8
+; SI-NEXT: v_writelane_b32 v22, s47, 9
; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 42
-; SI-NEXT: v_writelane_b32 v22, s47, 43
+; SI-NEXT: v_writelane_b32 v22, s46, 4
+; SI-NEXT: v_writelane_b32 v22, s47, 5
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 40
-; SI-NEXT: v_writelane_b32 v22, s47, 41
+; SI-NEXT: v_writelane_b32 v22, s46, 2
+; SI-NEXT: v_writelane_b32 v22, s47, 3
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 38
-; SI-NEXT: v_writelane_b32 v22, s47, 39
+; SI-NEXT: v_writelane_b32 v22, s46, 0
+; SI-NEXT: v_writelane_b32 v22, s47, 1
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 36
-; SI-NEXT: v_writelane_b32 v22, s47, 37
+; SI-NEXT: v_writelane_b32 v21, s46, 62
+; SI-NEXT: v_writelane_b32 v21, s47, 63
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 34
-; SI-NEXT: v_writelane_b32 v22, s47, 35
+; SI-NEXT: v_writelane_b32 v21, s46, 60
+; SI-NEXT: v_writelane_b32 v21, s47, 61
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 32
-; SI-NEXT: v_writelane_b32 v22, s47, 33
+; SI-NEXT: v_writelane_b32 v21, s46, 58
+; SI-NEXT: v_writelane_b32 v21, s47, 59
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 30
-; SI-NEXT: v_writelane_b32 v22, s47, 31
+; SI-NEXT: v_writelane_b32 v21, s46, 56
+; SI-NEXT: v_writelane_b32 v21, s47, 57
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 28
-; SI-NEXT: v_writelane_b32 v22, s47, 29
+; SI-NEXT: v_writelane_b32 v21, s46, 54
+; SI-NEXT: v_writelane_b32 v21, s47, 55
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 26
-; SI-NEXT: v_writelane_b32 v22, s47, 27
+; SI-NEXT: v_writelane_b32 v21, s46, 52
+; SI-NEXT: v_writelane_b32 v21, s47, 53
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 24
-; SI-NEXT: v_writelane_b32 v22, s47, 25
+; SI-NEXT: v_writelane_b32 v21, s46, 50
+; SI-NEXT: v_writelane_b32 v21, s47, 51
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 22
-; SI-NEXT: v_writelane_b32 v22, s47, 23
+; SI-NEXT: v_writelane_b32 v21, s46, 48
+; SI-NEXT: v_writelane_b32 v21, s47, 49
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 20
-; SI-NEXT: v_writelane_b32 v22, s47, 21
+; SI-NEXT: v_writelane_b32 v21, s46, 46
+; SI-NEXT: v_writelane_b32 v21, s47, 47
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 18
-; SI-NEXT: v_writelane_b32 v22, s47, 19
+; SI-NEXT: v_writelane_b32 v21, s46, 44
+; SI-NEXT: v_writelane_b32 v21, s47, 45
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 16
-; SI-NEXT: v_writelane_b32 v22, s47, 17
+; SI-NEXT: v_writelane_b32 v21, s46, 42
+; SI-NEXT: v_writelane_b32 v21, s47, 43
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 14
-; SI-NEXT: v_writelane_b32 v22, s47, 15
+; SI-NEXT: v_writelane_b32 v21, s46, 40
+; SI-NEXT: v_writelane_b32 v21, s47, 41
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 12
-; SI-NEXT: v_writelane_b32 v22, s47, 13
+; SI-NEXT: v_writelane_b32 v21, s46, 38
+; SI-NEXT: v_writelane_b32 v21, s47, 39
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 10
-; SI-NEXT: v_writelane_b32 v22, s47, 11
+; SI-NEXT: v_writelane_b32 v21, s46, 36
+; SI-NEXT: v_writelane_b32 v21, s47, 37
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 8
-; SI-NEXT: v_writelane_b32 v22, s47, 9
+; SI-NEXT: v_writelane_b32 v21, s46, 34
+; SI-NEXT: v_writelane_b32 v21, s47, 35
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 6
-; SI-NEXT: v_writelane_b32 v22, s47, 7
+; SI-NEXT: v_writelane_b32 v21, s46, 32
+; SI-NEXT: v_writelane_b32 v21, s47, 33
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 4
-; SI-NEXT: v_writelane_b32 v22, s47, 5
+; SI-NEXT: v_writelane_b32 v21, s46, 30
+; SI-NEXT: v_writelane_b32 v21, s47, 31
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 2
-; SI-NEXT: v_writelane_b32 v22, s47, 3
+; SI-NEXT: v_writelane_b32 v21, s46, 28
+; SI-NEXT: v_writelane_b32 v21, s47, 29
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 0
-; SI-NEXT: s_lshr_b32 s48, s23, 24
-; SI-NEXT: s_lshr_b32 s49, s23, 16
-; SI-NEXT: s_lshr_b32 s54, s23, 8
-; SI-NEXT: s_lshr_b32 s55, s21, 24
-; SI-NEXT: s_lshr_b32 s50, s21, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 8
-; SI-NEXT: s_lshr_b32 s52, s19, 24
-; SI-NEXT: s_lshr_b32 s64, s19, 16
-; SI-NEXT: s_lshr_b32 s65, s19, 8
-; SI-NEXT: s_lshr_b32 s66, s17, 24
-; SI-NEXT: s_lshr_b32 s67, s17, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 8
-; SI-NEXT: s_lshr_b64 s[68:69], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, s47, 1
-; SI-NEXT: s_lshr_b64 s[70:71], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[80:81], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[82:83], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[84:85], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[86:87], s[28:29], 16
-; SI-NEXT: s_lshr_b64 s[96:97], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[98:99], s[26:27], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 26
+; SI-NEXT: v_writelane_b32 v21, s47, 27
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 24
+; SI-NEXT: v_writelane_b32 v21, s47, 25
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
+; SI-NEXT: v_writelane_b32 v21, s46, 22
+; SI-NEXT: v_writelane_b32 v21, s47, 23
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 8
+; SI-NEXT: v_writelane_b32 v21, s46, 20
+; SI-NEXT: v_writelane_b32 v21, s47, 21
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 18
+; SI-NEXT: v_writelane_b32 v21, s47, 19
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
+; SI-NEXT: v_writelane_b32 v21, s46, 16
+; SI-NEXT: v_writelane_b32 v21, s47, 17
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 8
+; SI-NEXT: v_writelane_b32 v21, s46, 14
+; SI-NEXT: v_writelane_b32 v21, s47, 15
+; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 12
+; SI-NEXT: v_writelane_b32 v21, s47, 13
+; SI-NEXT: s_lshr_b64 s[76:77], s[26:27], 8
+; SI-NEXT: v_writelane_b32 v21, s76, 10
+; SI-NEXT: v_writelane_b32 v21, s77, 11
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v21, s76, 8
+; SI-NEXT: v_writelane_b32 v21, s77, 9
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 16
+; SI-NEXT: v_writelane_b32 v21, s76, 6
+; SI-NEXT: v_writelane_b32 v21, s77, 7
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 8
+; SI-NEXT: v_writelane_b32 v21, s76, 4
+; SI-NEXT: v_writelane_b32 v21, s77, 5
+; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 24
+; SI-NEXT: v_writelane_b32 v21, s76, 2
+; SI-NEXT: v_writelane_b32 v21, s77, 3
+; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v21, s76, 0
+; SI-NEXT: s_lshr_b32 s68, s5, 24
+; SI-NEXT: s_lshr_b32 s48, s5, 16
+; SI-NEXT: s_lshr_b32 s69, s5, 8
+; SI-NEXT: s_lshr_b32 s49, s7, 24
+; SI-NEXT: s_lshr_b32 s54, s7, 16
+; SI-NEXT: s_lshr_b32 s70, s7, 8
+; SI-NEXT: s_lshr_b32 s55, s9, 24
+; SI-NEXT: s_lshr_b32 s71, s9, 16
+; SI-NEXT: s_lshr_b32 s50, s9, 8
+; SI-NEXT: s_lshr_b32 s80, s11, 24
+; SI-NEXT: s_lshr_b32 s51, s11, 16
+; SI-NEXT: s_lshr_b32 s81, s11, 8
+; SI-NEXT: s_lshr_b32 s82, s13, 24
+; SI-NEXT: s_lshr_b32 s52, s13, 16
+; SI-NEXT: s_lshr_b32 s83, s13, 8
+; SI-NEXT: s_lshr_b32 s64, s15, 24
+; SI-NEXT: s_lshr_b32 s65, s15, 16
+; SI-NEXT: s_lshr_b32 s84, s15, 8
+; SI-NEXT: s_lshr_b32 s86, s41, 24
+; SI-NEXT: s_lshr_b32 s85, s41, 16
+; SI-NEXT: s_lshr_b32 s96, s41, 8
+; SI-NEXT: s_lshr_b32 s87, s43, 24
+; SI-NEXT: s_lshr_b32 s66, s43, 16
+; SI-NEXT: s_lshr_b32 s97, s43, 8
+; SI-NEXT: s_lshr_b32 s99, s45, 24
+; SI-NEXT: s_lshr_b32 s98, s45, 16
+; SI-NEXT: s_lshr_b32 s67, s45, 8
+; SI-NEXT: s_lshr_b32 s53, s29, 24
+; SI-NEXT: s_lshr_b32 s56, s29, 16
+; SI-NEXT: s_lshr_b32 s57, s29, 8
+; SI-NEXT: s_lshr_b32 s58, s27, 24
+; SI-NEXT: s_lshr_b32 s60, s27, 16
+; SI-NEXT: s_lshr_b32 s59, s27, 8
+; SI-NEXT: s_lshr_b32 s62, s25, 24
+; SI-NEXT: s_lshr_b32 s61, s25, 16
+; SI-NEXT: s_lshr_b32 s72, s25, 8
+; SI-NEXT: s_lshr_b32 s63, s23, 24
+; SI-NEXT: s_lshr_b32 s74, s23, 16
+; SI-NEXT: s_lshr_b32 s73, s23, 8
+; SI-NEXT: s_lshr_b32 s75, s21, 16
; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[60:61], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v21, s77, 1
; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 8
; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 24
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
@@ -77906,11 +78436,232 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: s_mov_b64 s[38:39], -1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr97
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_writelane_b32 v21, s46, 0
+; SI-NEXT: v_writelane_b32 v21, s47, 1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s46, 2
+; SI-NEXT: v_writelane_b32 v21, s47, 3
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s46, 4
+; SI-NEXT: v_writelane_b32 v21, s47, 5
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s46, 6
+; SI-NEXT: v_writelane_b32 v21, s47, 7
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s46, 8
+; SI-NEXT: v_writelane_b32 v21, s47, 9
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s46, 10
+; SI-NEXT: v_writelane_b32 v21, s47, 11
+; SI-NEXT: v_writelane_b32 v21, s98, 12
+; SI-NEXT: v_writelane_b32 v21, s99, 13
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v21, s98, 14
+; SI-NEXT: v_writelane_b32 v21, s99, 15
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 16
+; SI-NEXT: v_writelane_b32 v21, s99, 17
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 18
+; SI-NEXT: v_writelane_b32 v21, s99, 19
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 20
+; SI-NEXT: v_writelane_b32 v21, s99, 21
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 22
+; SI-NEXT: v_writelane_b32 v21, s99, 23
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 24
+; SI-NEXT: v_writelane_b32 v21, s99, 25
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 26
+; SI-NEXT: v_writelane_b32 v21, s99, 27
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 28
+; SI-NEXT: v_writelane_b32 v21, s99, 29
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 30
+; SI-NEXT: v_writelane_b32 v21, s99, 31
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 32
+; SI-NEXT: v_writelane_b32 v21, s99, 33
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 34
+; SI-NEXT: v_writelane_b32 v21, s99, 35
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 36
+; SI-NEXT: v_writelane_b32 v21, s99, 37
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 38
+; SI-NEXT: v_writelane_b32 v21, s99, 39
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 40
+; SI-NEXT: v_writelane_b32 v21, s99, 41
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 42
+; SI-NEXT: v_writelane_b32 v21, s99, 43
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 44
+; SI-NEXT: v_writelane_b32 v21, s99, 45
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 46
+; SI-NEXT: v_writelane_b32 v21, s99, 47
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 48
+; SI-NEXT: v_writelane_b32 v21, s99, 49
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 50
+; SI-NEXT: v_writelane_b32 v21, s99, 51
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 52
+; SI-NEXT: v_writelane_b32 v21, s99, 53
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 54
+; SI-NEXT: v_writelane_b32 v21, s99, 55
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 56
+; SI-NEXT: v_writelane_b32 v21, s99, 57
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 58
+; SI-NEXT: v_writelane_b32 v21, s99, 59
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 60
+; SI-NEXT: v_writelane_b32 v21, s99, 61
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v21, s98, 62
+; SI-NEXT: v_writelane_b32 v21, s99, 63
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v22, s98, 0
+; SI-NEXT: v_writelane_b32 v22, s99, 1
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v22, s98, 2
+; SI-NEXT: v_writelane_b32 v22, s99, 3
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v22, s98, 4
+; SI-NEXT: v_writelane_b32 v22, s99, 5
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v22, s98, 6
+; SI-NEXT: v_writelane_b32 v22, s99, 7
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: v_writelane_b32 v22, s98, 8
+; SI-NEXT: v_writelane_b32 v22, s99, 9
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[38:39], s[38:39], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_mov_b32 s38, s68
+; SI-NEXT: s_mov_b32 s39, s69
+; SI-NEXT: s_mov_b32 s68, s48
+; SI-NEXT: s_mov_b32 s48, s49
+; SI-NEXT: s_mov_b32 s49, s54
+; SI-NEXT: s_mov_b32 s54, s70
+; SI-NEXT: s_mov_b32 s69, s71
+; SI-NEXT: s_mov_b32 s70, s55
+; SI-NEXT: s_mov_b32 s55, s50
+; SI-NEXT: s_mov_b32 s50, s80
+; SI-NEXT: s_mov_b32 s71, s81
+; SI-NEXT: s_mov_b32 s80, s51
+; SI-NEXT: s_mov_b32 s51, s82
+; SI-NEXT: s_mov_b32 s81, s83
+; SI-NEXT: s_mov_b32 s82, s52
+; SI-NEXT: s_mov_b32 s52, s64
+; SI-NEXT: s_mov_b32 s64, s65
+; SI-NEXT: s_mov_b32 s65, s84
+; SI-NEXT: s_mov_b32 s83, s85
+; SI-NEXT: s_mov_b32 s84, s86
+; SI-NEXT: s_mov_b32 s85, s87
+; SI-NEXT: s_mov_b32 s86, s96
+; SI-NEXT: s_mov_b32 s87, s97
+; SI-NEXT: s_mov_b32 s96, s66
+; SI-NEXT: s_mov_b32 s66, s99
+; SI-NEXT: s_mov_b32 s97, s67
+; SI-NEXT: s_mov_b32 s67, s53
+; SI-NEXT: s_mov_b32 s53, s56
+; SI-NEXT: s_mov_b32 s56, s57
+; SI-NEXT: s_mov_b32 s99, s58
+; SI-NEXT: s_mov_b32 s58, s59
+; SI-NEXT: s_mov_b32 s59, s60
+; SI-NEXT: v_readlane_b32 s60, v22, 10
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -77943,174 +78694,173 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: s_lshr_b32 s46, s5, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 46
-; SI-NEXT: s_lshr_b32 s46, s5, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 47
-; SI-NEXT: s_lshr_b32 s46, s5, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 48
-; SI-NEXT: s_lshr_b32 s46, s7, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 49
-; SI-NEXT: s_lshr_b32 s46, s7, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 50
-; SI-NEXT: s_lshr_b32 s46, s7, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 51
-; SI-NEXT: s_lshr_b32 s46, s9, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 52
-; SI-NEXT: s_lshr_b32 s46, s9, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 53
-; SI-NEXT: s_lshr_b32 s46, s9, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 54
-; SI-NEXT: s_lshr_b32 s46, s11, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 55
-; SI-NEXT: s_lshr_b32 s46, s11, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 56
-; SI-NEXT: s_lshr_b32 s46, s11, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 57
-; SI-NEXT: s_lshr_b32 s46, s13, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 58
-; SI-NEXT: s_lshr_b32 s46, s13, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 59
-; SI-NEXT: s_lshr_b32 s46, s13, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 60
-; SI-NEXT: s_lshr_b32 s46, s15, 24
-; SI-NEXT: v_writelane_b32 v22, s46, 61
-; SI-NEXT: s_lshr_b32 s46, s15, 16
-; SI-NEXT: v_writelane_b32 v22, s46, 62
-; SI-NEXT: s_lshr_b32 s46, s15, 8
-; SI-NEXT: v_writelane_b32 v22, s46, 63
-; SI-NEXT: s_lshr_b32 s46, s41, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 0
-; SI-NEXT: s_lshr_b32 s46, s41, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 1
-; SI-NEXT: s_lshr_b32 s46, s41, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 2
-; SI-NEXT: s_lshr_b32 s46, s43, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 3
-; SI-NEXT: s_lshr_b32 s46, s43, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 4
-; SI-NEXT: s_lshr_b32 s46, s43, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 5
-; SI-NEXT: s_lshr_b32 s46, s45, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 6
-; SI-NEXT: s_lshr_b32 s46, s45, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 7
-; SI-NEXT: s_lshr_b32 s46, s45, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 8
-; SI-NEXT: s_lshr_b32 s46, s29, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 9
-; SI-NEXT: s_lshr_b32 s46, s29, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 10
-; SI-NEXT: s_lshr_b32 s46, s29, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 11
-; SI-NEXT: s_lshr_b32 s46, s27, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 12
-; SI-NEXT: s_lshr_b32 s46, s27, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 13
-; SI-NEXT: s_lshr_b32 s46, s27, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 14
-; SI-NEXT: s_lshr_b32 s46, s25, 24
-; SI-NEXT: v_writelane_b32 v21, s46, 15
-; SI-NEXT: s_lshr_b32 s46, s25, 16
-; SI-NEXT: v_writelane_b32 v21, s46, 16
-; SI-NEXT: s_lshr_b32 s46, s25, 8
-; SI-NEXT: v_writelane_b32 v21, s46, 17
+; SI-NEXT: s_lshr_b32 s46, s21, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 11
+; SI-NEXT: s_lshr_b32 s46, s19, 24
+; SI-NEXT: v_writelane_b32 v22, s46, 12
+; SI-NEXT: s_lshr_b32 s46, s19, 16
+; SI-NEXT: v_writelane_b32 v22, s46, 13
+; SI-NEXT: s_lshr_b32 s46, s19, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 14
+; SI-NEXT: s_lshr_b32 s46, s17, 24
+; SI-NEXT: v_writelane_b32 v22, s46, 15
+; SI-NEXT: s_lshr_b32 s46, s17, 16
+; SI-NEXT: v_writelane_b32 v22, s46, 16
+; SI-NEXT: s_lshr_b32 s46, s17, 8
+; SI-NEXT: v_writelane_b32 v22, s46, 17
; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 44
-; SI-NEXT: v_writelane_b32 v22, s47, 45
+; SI-NEXT: v_writelane_b32 v22, s46, 6
+; SI-NEXT: v_writelane_b32 v22, s47, 7
+; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
+; SI-NEXT: v_writelane_b32 v22, s46, 8
+; SI-NEXT: v_writelane_b32 v22, s47, 9
; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 42
-; SI-NEXT: v_writelane_b32 v22, s47, 43
+; SI-NEXT: v_writelane_b32 v22, s46, 4
+; SI-NEXT: v_writelane_b32 v22, s47, 5
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 40
-; SI-NEXT: v_writelane_b32 v22, s47, 41
+; SI-NEXT: v_writelane_b32 v22, s46, 2
+; SI-NEXT: v_writelane_b32 v22, s47, 3
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 38
-; SI-NEXT: v_writelane_b32 v22, s47, 39
+; SI-NEXT: v_writelane_b32 v22, s46, 0
+; SI-NEXT: v_writelane_b32 v22, s47, 1
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 36
-; SI-NEXT: v_writelane_b32 v22, s47, 37
+; SI-NEXT: v_writelane_b32 v21, s46, 62
+; SI-NEXT: v_writelane_b32 v21, s47, 63
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 34
-; SI-NEXT: v_writelane_b32 v22, s47, 35
+; SI-NEXT: v_writelane_b32 v21, s46, 60
+; SI-NEXT: v_writelane_b32 v21, s47, 61
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 32
-; SI-NEXT: v_writelane_b32 v22, s47, 33
+; SI-NEXT: v_writelane_b32 v21, s46, 58
+; SI-NEXT: v_writelane_b32 v21, s47, 59
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 30
-; SI-NEXT: v_writelane_b32 v22, s47, 31
+; SI-NEXT: v_writelane_b32 v21, s46, 56
+; SI-NEXT: v_writelane_b32 v21, s47, 57
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 28
-; SI-NEXT: v_writelane_b32 v22, s47, 29
+; SI-NEXT: v_writelane_b32 v21, s46, 54
+; SI-NEXT: v_writelane_b32 v21, s47, 55
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 26
-; SI-NEXT: v_writelane_b32 v22, s47, 27
+; SI-NEXT: v_writelane_b32 v21, s46, 52
+; SI-NEXT: v_writelane_b32 v21, s47, 53
; SI-NEXT: s_lshr_b64 s[46:47], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 24
-; SI-NEXT: v_writelane_b32 v22, s47, 25
+; SI-NEXT: v_writelane_b32 v21, s46, 50
+; SI-NEXT: v_writelane_b32 v21, s47, 51
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 22
-; SI-NEXT: v_writelane_b32 v22, s47, 23
+; SI-NEXT: v_writelane_b32 v21, s46, 48
+; SI-NEXT: v_writelane_b32 v21, s47, 49
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 20
-; SI-NEXT: v_writelane_b32 v22, s47, 21
+; SI-NEXT: v_writelane_b32 v21, s46, 46
+; SI-NEXT: v_writelane_b32 v21, s47, 47
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 18
-; SI-NEXT: v_writelane_b32 v22, s47, 19
+; SI-NEXT: v_writelane_b32 v21, s46, 44
+; SI-NEXT: v_writelane_b32 v21, s47, 45
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 16
-; SI-NEXT: v_writelane_b32 v22, s47, 17
+; SI-NEXT: v_writelane_b32 v21, s46, 42
+; SI-NEXT: v_writelane_b32 v21, s47, 43
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 14
-; SI-NEXT: v_writelane_b32 v22, s47, 15
+; SI-NEXT: v_writelane_b32 v21, s46, 40
+; SI-NEXT: v_writelane_b32 v21, s47, 41
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 12
-; SI-NEXT: v_writelane_b32 v22, s47, 13
+; SI-NEXT: v_writelane_b32 v21, s46, 38
+; SI-NEXT: v_writelane_b32 v21, s47, 39
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 10
-; SI-NEXT: v_writelane_b32 v22, s47, 11
+; SI-NEXT: v_writelane_b32 v21, s46, 36
+; SI-NEXT: v_writelane_b32 v21, s47, 37
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 8
-; SI-NEXT: v_writelane_b32 v22, s47, 9
+; SI-NEXT: v_writelane_b32 v21, s46, 34
+; SI-NEXT: v_writelane_b32 v21, s47, 35
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 6
-; SI-NEXT: v_writelane_b32 v22, s47, 7
+; SI-NEXT: v_writelane_b32 v21, s46, 32
+; SI-NEXT: v_writelane_b32 v21, s47, 33
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 24
-; SI-NEXT: v_writelane_b32 v22, s46, 4
-; SI-NEXT: v_writelane_b32 v22, s47, 5
+; SI-NEXT: v_writelane_b32 v21, s46, 30
+; SI-NEXT: v_writelane_b32 v21, s47, 31
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v22, s46, 2
-; SI-NEXT: v_writelane_b32 v22, s47, 3
+; SI-NEXT: v_writelane_b32 v21, s46, 28
+; SI-NEXT: v_writelane_b32 v21, s47, 29
; SI-NEXT: s_lshr_b64 s[46:47], s[42:43], 8
-; SI-NEXT: v_writelane_b32 v22, s46, 0
-; SI-NEXT: s_lshr_b32 s48, s23, 24
-; SI-NEXT: s_lshr_b32 s49, s23, 16
-; SI-NEXT: s_lshr_b32 s54, s23, 8
-; SI-NEXT: s_lshr_b32 s55, s21, 24
-; SI-NEXT: s_lshr_b32 s50, s21, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 8
-; SI-NEXT: s_lshr_b32 s52, s19, 24
-; SI-NEXT: s_lshr_b32 s64, s19, 16
-; SI-NEXT: s_lshr_b32 s65, s19, 8
-; SI-NEXT: s_lshr_b32 s66, s17, 24
-; SI-NEXT: s_lshr_b32 s67, s17, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 8
-; SI-NEXT: s_lshr_b64 s[68:69], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, s47, 1
-; SI-NEXT: s_lshr_b64 s[70:71], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[80:81], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[82:83], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[84:85], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[86:87], s[28:29], 16
-; SI-NEXT: s_lshr_b64 s[96:97], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[98:99], s[26:27], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 26
+; SI-NEXT: v_writelane_b32 v21, s47, 27
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 24
+; SI-NEXT: v_writelane_b32 v21, s47, 25
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 16
+; SI-NEXT: v_writelane_b32 v21, s46, 22
+; SI-NEXT: v_writelane_b32 v21, s47, 23
+; SI-NEXT: s_lshr_b64 s[46:47], s[44:45], 8
+; SI-NEXT: v_writelane_b32 v21, s46, 20
+; SI-NEXT: v_writelane_b32 v21, s47, 21
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 18
+; SI-NEXT: v_writelane_b32 v21, s47, 19
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 16
+; SI-NEXT: v_writelane_b32 v21, s46, 16
+; SI-NEXT: v_writelane_b32 v21, s47, 17
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 8
+; SI-NEXT: v_writelane_b32 v21, s46, 14
+; SI-NEXT: v_writelane_b32 v21, s47, 15
+; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 24
+; SI-NEXT: v_writelane_b32 v21, s46, 12
+; SI-NEXT: v_writelane_b32 v21, s47, 13
+; SI-NEXT: s_lshr_b64 s[76:77], s[26:27], 8
+; SI-NEXT: v_writelane_b32 v21, s76, 10
+; SI-NEXT: v_writelane_b32 v21, s77, 11
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v21, s76, 8
+; SI-NEXT: v_writelane_b32 v21, s77, 9
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 16
+; SI-NEXT: v_writelane_b32 v21, s76, 6
+; SI-NEXT: v_writelane_b32 v21, s77, 7
+; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 8
+; SI-NEXT: v_writelane_b32 v21, s76, 4
+; SI-NEXT: v_writelane_b32 v21, s77, 5
+; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 24
+; SI-NEXT: v_writelane_b32 v21, s76, 2
+; SI-NEXT: v_writelane_b32 v21, s77, 3
+; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v21, s76, 0
+; SI-NEXT: s_lshr_b32 s38, s5, 24
+; SI-NEXT: s_lshr_b32 s68, s5, 16
+; SI-NEXT: s_lshr_b32 s39, s5, 8
+; SI-NEXT: s_lshr_b32 s48, s7, 24
+; SI-NEXT: s_lshr_b32 s49, s7, 16
+; SI-NEXT: s_lshr_b32 s54, s7, 8
+; SI-NEXT: s_lshr_b32 s70, s9, 24
+; SI-NEXT: s_lshr_b32 s69, s9, 16
+; SI-NEXT: s_lshr_b32 s55, s9, 8
+; SI-NEXT: s_lshr_b32 s50, s11, 24
+; SI-NEXT: s_lshr_b32 s80, s11, 16
+; SI-NEXT: s_lshr_b32 s71, s11, 8
+; SI-NEXT: s_lshr_b32 s51, s13, 24
+; SI-NEXT: s_lshr_b32 s82, s13, 16
+; SI-NEXT: s_lshr_b32 s81, s13, 8
+; SI-NEXT: s_lshr_b32 s52, s15, 24
+; SI-NEXT: s_lshr_b32 s64, s15, 16
+; SI-NEXT: s_lshr_b32 s65, s15, 8
+; SI-NEXT: s_lshr_b32 s84, s41, 24
+; SI-NEXT: s_lshr_b32 s83, s41, 16
+; SI-NEXT: s_lshr_b32 s86, s41, 8
+; SI-NEXT: s_lshr_b32 s85, s43, 24
+; SI-NEXT: s_lshr_b32 s96, s43, 16
+; SI-NEXT: s_lshr_b32 s87, s43, 8
+; SI-NEXT: s_lshr_b32 s66, s45, 24
+; SI-NEXT: s_lshr_b32 s98, s45, 16
+; SI-NEXT: s_lshr_b32 s97, s45, 8
+; SI-NEXT: s_lshr_b32 s67, s29, 24
+; SI-NEXT: s_lshr_b32 s53, s29, 16
+; SI-NEXT: s_lshr_b32 s56, s29, 8
+; SI-NEXT: s_lshr_b32 s99, s27, 24
+; SI-NEXT: s_lshr_b32 s59, s27, 16
+; SI-NEXT: s_lshr_b32 s58, s27, 8
+; SI-NEXT: s_lshr_b32 s62, s25, 24
+; SI-NEXT: s_lshr_b32 s61, s25, 16
+; SI-NEXT: s_lshr_b32 s72, s25, 8
+; SI-NEXT: s_lshr_b32 s63, s23, 24
+; SI-NEXT: s_lshr_b32 s74, s23, 16
+; SI-NEXT: s_lshr_b32 s73, s23, 8
+; SI-NEXT: s_lshr_b32 s60, s21, 24
+; SI-NEXT: s_lshr_b32 s75, s21, 16
; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[60:61], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v21, s77, 1
; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 8
; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 24
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
@@ -78118,26 +78868,29 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[16:17], 8
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s47, s38, 8
+; SI-NEXT: s_lshl_b32 s47, s36, 8
; SI-NEXT: s_or_b32 s16, s16, s47
-; SI-NEXT: s_and_b32 s47, s36, 0xff
+; SI-NEXT: s_and_b32 s47, s34, 0xff
; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_lshl_b32 s57, s34, 24
+; SI-NEXT: s_lshl_b32 s57, vcc_lo, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s47, s57, s47
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s17, 0xff
-; SI-NEXT: s_lshl_b32 s17, s53, 8
+; SI-NEXT: v_readlane_b32 s17, v22, 17
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s67, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s47, v22, 15
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s47, s66, 24
+; SI-NEXT: s_lshl_b32 s47, s47, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s47, s17
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -78154,15 +78907,18 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: v_readlane_b32 s17, v22, 14
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s19, 0xff
-; SI-NEXT: s_lshl_b32 s17, s65, 8
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s64, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 13
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 12
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s52, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 8, v0
@@ -78180,28 +78936,32 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 12, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: v_readlane_b32 s17, v22, 11
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s21, 0xff
-; SI-NEXT: s_lshl_b32 s17, s51, 8
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s50, 0xff
+; SI-NEXT: s_and_b32 s17, s75, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s55, 24
+; SI-NEXT: s_lshl_b32 s18, s60, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: v_readlane_b32 s18, v21, 0
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s22, 0xff
; SI-NEXT: s_lshl_b32 s17, s76, 8
+; SI-NEXT: v_readlane_b32 s19, v21, 1
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s74, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s18, v21, 2
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s72, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 20, v0
@@ -78210,196 +78970,201 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s23, 0xff
-; SI-NEXT: s_lshl_b32 s17, s54, 8
+; SI-NEXT: s_lshl_b32 s17, s73, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s49, 0xff
+; SI-NEXT: s_and_b32 s17, s74, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 3
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s48, 24
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s63, 24
; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v21, 4
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s19, v21, 5
; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_lshl_b32 s17, s18, 8
+; SI-NEXT: v_readlane_b32 s18, v21, 6
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s24, 0xff
-; SI-NEXT: s_lshl_b32 s17, s62, 8
+; SI-NEXT: v_readlane_b32 s19, v21, 7
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s60, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s18, v21, 8
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s58, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 28, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 17
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s25, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s72, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 16
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 15
+; SI-NEXT: s_and_b32 s17, s61, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s62, 24
+; SI-NEXT: v_readlane_b32 s19, v21, 9
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: v_readlane_b32 s18, v21, 10
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s26, 0xff
-; SI-NEXT: s_lshl_b32 s17, s56, 8
+; SI-NEXT: v_readlane_b32 s19, v21, 11
+; SI-NEXT: s_lshl_b32 s17, s18, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s46, 0xff
+; SI-NEXT: v_readlane_b32 s18, v21, 12
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s98, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 36, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 14
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s27, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s58, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 13
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 12
+; SI-NEXT: s_and_b32 s17, s59, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 13
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s99, 24
; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v21, 14
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s19, v21, 15
; SI-NEXT: v_add_i32_e32 v1, vcc, 40, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_lshl_b32 s17, s18, 8
+; SI-NEXT: v_readlane_b32 s18, v21, 16
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s28, 0xff
-; SI-NEXT: s_lshl_b32 s17, s96, 8
+; SI-NEXT: v_readlane_b32 s19, v21, 17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s86, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s18, v21, 18
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s84, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 44, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 11
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s29, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s56, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 10
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 9
+; SI-NEXT: s_and_b32 s17, s53, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 19
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s67, 24
; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v21, 20
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: v_readlane_b32 s19, v21, 21
; SI-NEXT: v_add_i32_e32 v1, vcc, 48, v0
; SI-NEXT: s_or_b32 s16, s16, s17
+; SI-NEXT: s_lshl_b32 s17, s18, 8
+; SI-NEXT: v_readlane_b32 s18, v21, 22
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: s_lshl_b32 s17, s82, 8
+; SI-NEXT: v_readlane_b32 s19, v21, 23
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s80, 0xff
+; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s18, v21, 24
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s70, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 52, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 8
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s45, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s97, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 7
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 6
+; SI-NEXT: s_and_b32 s17, s98, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 25
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s66, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 0
+; SI-NEXT: v_readlane_b32 s18, v21, 26
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 1
+; SI-NEXT: v_readlane_b32 s19, v21, 27
; SI-NEXT: v_add_i32_e32 v1, vcc, 56, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 2
+; SI-NEXT: v_readlane_b32 s18, v21, 28
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 3
+; SI-NEXT: v_readlane_b32 s19, v21, 29
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 4
+; SI-NEXT: v_readlane_b32 s18, v21, 30
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 60, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 5
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s43, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s87, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 4
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 3
-; SI-NEXT: v_readlane_b32 s19, v22, 5
+; SI-NEXT: s_and_b32 s17, s96, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 31
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s85, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 6
+; SI-NEXT: v_readlane_b32 s18, v21, 32
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 7
+; SI-NEXT: v_readlane_b32 s19, v21, 33
; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 8
+; SI-NEXT: v_readlane_b32 s18, v21, 34
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 9
+; SI-NEXT: v_readlane_b32 s19, v21, 35
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 10
+; SI-NEXT: v_readlane_b32 s18, v21, 36
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x44, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 2
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s41, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s86, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v21, 1
-; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 0
+; SI-NEXT: s_and_b32 s17, s83, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s84, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x48, v0
@@ -78407,16 +79172,16 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
-; SI-NEXT: v_readlane_b32 s16, v22, 12
+; SI-NEXT: v_readlane_b32 s16, v21, 38
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v22, 13
+; SI-NEXT: v_readlane_b32 s17, v21, 39
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 11
+; SI-NEXT: v_readlane_b32 s19, v21, 37
; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: v_readlane_b32 s16, v22, 14
-; SI-NEXT: v_readlane_b32 s17, v22, 15
+; SI-NEXT: v_readlane_b32 s16, v21, 40
+; SI-NEXT: v_readlane_b32 s17, v21, 41
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 16
+; SI-NEXT: v_readlane_b32 s18, v21, 42
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_lshl_b32 s17, s18, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
@@ -78427,14 +79192,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: s_and_b32 s14, s15, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 63
-; SI-NEXT: s_lshl_b32 s15, s15, 8
+; SI-NEXT: s_lshl_b32 s15, s65, 8
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: v_readlane_b32 s15, v22, 62
-; SI-NEXT: s_and_b32 s15, s15, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 61
+; SI-NEXT: s_and_b32 s15, s64, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_lshl_b32 s16, s16, 24
+; SI-NEXT: s_lshl_b32 s16, s52, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x50, v0
@@ -78442,15 +79204,15 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_readlane_b32 s14, v22, 18
+; SI-NEXT: v_readlane_b32 s14, v21, 44
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 19
+; SI-NEXT: v_readlane_b32 s15, v21, 45
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: v_readlane_b32 s14, v22, 20
-; SI-NEXT: v_readlane_b32 s15, v22, 21
+; SI-NEXT: v_readlane_b32 s14, v21, 46
+; SI-NEXT: v_readlane_b32 s15, v21, 47
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 22
+; SI-NEXT: v_readlane_b32 s16, v21, 48
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s16, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
@@ -78461,14 +79223,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: s_and_b32 s12, s13, 0xff
-; SI-NEXT: v_readlane_b32 s13, v22, 60
-; SI-NEXT: s_lshl_b32 s13, s13, 8
+; SI-NEXT: s_lshl_b32 s13, s81, 8
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: v_readlane_b32 s13, v22, 59
-; SI-NEXT: s_and_b32 s13, s13, 0xff
-; SI-NEXT: v_readlane_b32 s14, v22, 58
+; SI-NEXT: s_and_b32 s13, s82, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s14, s14, 24
+; SI-NEXT: s_lshl_b32 s14, s51, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x58, v0
@@ -78476,15 +79235,15 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
-; SI-NEXT: v_readlane_b32 s12, v22, 24
+; SI-NEXT: v_readlane_b32 s12, v21, 50
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v22, 25
+; SI-NEXT: v_readlane_b32 s13, v21, 51
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: v_readlane_b32 s12, v22, 26
-; SI-NEXT: v_readlane_b32 s13, v22, 27
+; SI-NEXT: v_readlane_b32 s12, v21, 52
+; SI-NEXT: v_readlane_b32 s13, v21, 53
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v22, 28
+; SI-NEXT: v_readlane_b32 s14, v21, 54
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_lshl_b32 s13, s14, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
@@ -78495,14 +79254,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: s_and_b32 s10, s11, 0xff
-; SI-NEXT: v_readlane_b32 s11, v22, 57
-; SI-NEXT: s_lshl_b32 s11, s11, 8
+; SI-NEXT: s_lshl_b32 s11, s71, 8
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: v_readlane_b32 s11, v22, 56
-; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: v_readlane_b32 s12, v22, 55
+; SI-NEXT: s_and_b32 s11, s80, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_lshl_b32 s12, s50, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x60, v0
@@ -78510,15 +79266,15 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_readlane_b32 s10, v22, 30
+; SI-NEXT: v_readlane_b32 s10, v21, 56
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v22, 31
+; SI-NEXT: v_readlane_b32 s11, v21, 57
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: v_readlane_b32 s10, v22, 32
-; SI-NEXT: v_readlane_b32 s11, v22, 33
+; SI-NEXT: v_readlane_b32 s10, v21, 58
+; SI-NEXT: v_readlane_b32 s11, v21, 59
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v22, 34
+; SI-NEXT: v_readlane_b32 s12, v21, 60
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s11, s12, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
@@ -78529,14 +79285,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: s_and_b32 s8, s9, 0xff
-; SI-NEXT: v_readlane_b32 s9, v22, 54
-; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_lshl_b32 s9, s55, 8
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: v_readlane_b32 s9, v22, 53
-; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: v_readlane_b32 s10, v22, 52
+; SI-NEXT: s_and_b32 s9, s69, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_lshl_b32 s10, s70, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x68, v0
@@ -78544,15 +79297,15 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_readlane_b32 s8, v22, 36
+; SI-NEXT: v_readlane_b32 s8, v21, 62
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v22, 37
+; SI-NEXT: v_readlane_b32 s9, v21, 63
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: v_readlane_b32 s8, v22, 38
-; SI-NEXT: v_readlane_b32 s9, v22, 39
+; SI-NEXT: v_readlane_b32 s8, v22, 0
+; SI-NEXT: v_readlane_b32 s9, v22, 1
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v22, 40
+; SI-NEXT: v_readlane_b32 s10, v22, 2
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s10, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
@@ -78563,14 +79316,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_and_b32 s6, s7, 0xff
-; SI-NEXT: v_readlane_b32 s7, v22, 51
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_lshl_b32 s7, s54, 8
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: v_readlane_b32 s7, v22, 50
-; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 49
+; SI-NEXT: s_and_b32 s7, s49, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_lshl_b32 s8, s48, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x70, v0
@@ -78578,13 +79328,15 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: v_readlane_b32 s6, v22, 42
+; SI-NEXT: v_readlane_b32 s6, v22, 4
; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: v_readlane_b32 s7, v22, 5
; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: v_readlane_b32 s7, v22, 43
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s68, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 44
+; SI-NEXT: v_readlane_b32 s6, v22, 8
+; SI-NEXT: v_readlane_b32 s7, v22, 9
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: v_readlane_b32 s8, v22, 6
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s7, s8, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -78595,14 +79347,11 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_and_b32 s4, s5, 0xff
-; SI-NEXT: v_readlane_b32 s5, v22, 48
-; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_lshl_b32 s5, s39, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readlane_b32 s5, v22, 47
-; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: v_readlane_b32 s6, v22, 46
+; SI-NEXT: s_and_b32 s5, s68, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_lshl_b32 s6, s38, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x78, v0
@@ -78611,12 +79360,12 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v22, 17
-; SI-NEXT: v_readlane_b32 s17, v22, 23
-; SI-NEXT: v_readlane_b32 s15, v22, 29
-; SI-NEXT: v_readlane_b32 s13, v22, 35
-; SI-NEXT: v_readlane_b32 s11, v22, 41
-; SI-NEXT: v_readlane_b32 s9, v22, 45
+; SI-NEXT: v_readlane_b32 s19, v21, 43
+; SI-NEXT: v_readlane_b32 s17, v21, 49
+; SI-NEXT: v_readlane_b32 s15, v21, 55
+; SI-NEXT: v_readlane_b32 s13, v21, 61
+; SI-NEXT: v_readlane_b32 s11, v22, 3
+; SI-NEXT: v_readlane_b32 s9, v22, 7
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -78660,187 +79409,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s68, 0
-; SI-NEXT: v_writelane_b32 v22, s69, 1
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 2
-; SI-NEXT: v_writelane_b32 v22, s69, 3
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 4
-; SI-NEXT: v_writelane_b32 v22, s69, 5
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 6
-; SI-NEXT: v_writelane_b32 v22, s69, 7
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 8
-; SI-NEXT: v_writelane_b32 v22, s69, 9
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 10
-; SI-NEXT: v_writelane_b32 v22, s69, 11
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 12
-; SI-NEXT: v_writelane_b32 v22, s69, 13
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 14
-; SI-NEXT: v_writelane_b32 v22, s69, 15
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 16
-; SI-NEXT: v_writelane_b32 v22, s69, 17
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 18
-; SI-NEXT: v_writelane_b32 v22, s69, 19
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 20
-; SI-NEXT: v_writelane_b32 v22, s69, 21
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 22
-; SI-NEXT: v_writelane_b32 v22, s69, 23
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 24
-; SI-NEXT: v_writelane_b32 v22, s69, 25
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 26
-; SI-NEXT: v_writelane_b32 v22, s69, 27
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 28
-; SI-NEXT: v_writelane_b32 v22, s69, 29
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 30
-; SI-NEXT: v_writelane_b32 v22, s69, 31
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 32
-; SI-NEXT: v_writelane_b32 v22, s69, 33
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 34
-; SI-NEXT: v_writelane_b32 v22, s69, 35
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v22, s68, 36
-; SI-NEXT: v_writelane_b32 v22, s69, 37
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: v_writelane_b32 v22, s68, 38
-; SI-NEXT: v_writelane_b32 v22, s69, 39
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: v_writelane_b32 v22, s68, 40
-; SI-NEXT: v_writelane_b32 v22, s69, 41
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: v_writelane_b32 v22, s68, 42
-; SI-NEXT: v_writelane_b32 v22, s69, 43
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: v_writelane_b32 v22, s68, 44
-; SI-NEXT: v_writelane_b32 v22, s69, 45
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v16i64_to_v128i8_scalar:
; VI: ; %bb.0:
@@ -78902,164 +79470,336 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_cmp_lg_u32 s44, 0
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s5, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s5, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s4, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s4, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s7, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s7, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 8
-; VI-NEXT: s_lshr_b32 s46, s7, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s6, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s6, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 11
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s9, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 13
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s8, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s11, 16
; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s11, 8
+; VI-NEXT: s_lshr_b32 s46, s4, 16
; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s10, 16
+; VI-NEXT: s_lshr_b32 s46, s7, 24
; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s10, 8
+; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s13, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s12, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s15, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s15, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s14, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s14, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s41, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s41, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s41, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s40, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 35
-; VI-NEXT: s_lshr_b32 s46, s40, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s43, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s43, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s43, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s42, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s42, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s29, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s28, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s27, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s26, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s25, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b64 s[56:57], s[4:5], 24
+; VI-NEXT: s_lshr_b32 s57, s45, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 22
+; VI-NEXT: s_lshr_b32 s57, s44, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 23
+; VI-NEXT: s_lshr_b32 s57, s44, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 24
+; VI-NEXT: s_lshr_b32 s57, s29, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 25
+; VI-NEXT: s_lshr_b32 s57, s29, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 26
+; VI-NEXT: s_lshr_b32 s56, s29, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 27
+; VI-NEXT: s_lshr_b32 s57, s28, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 28
+; VI-NEXT: s_lshr_b32 s56, s28, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 29
+; VI-NEXT: s_lshr_b32 s57, s27, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 30
+; VI-NEXT: s_lshr_b32 s57, s27, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 31
+; VI-NEXT: s_lshr_b32 s56, s27, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 32
+; VI-NEXT: s_lshr_b32 s57, s26, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 33
+; VI-NEXT: s_lshr_b32 s56, s26, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 34
+; VI-NEXT: s_lshr_b32 s57, s25, 24
+; VI-NEXT: v_writelane_b32 v33, s57, 35
+; VI-NEXT: s_lshr_b32 s57, s25, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 36
+; VI-NEXT: s_lshr_b32 s56, s25, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 37
+; VI-NEXT: s_lshr_b32 s57, s24, 16
+; VI-NEXT: v_writelane_b32 v33, s57, 38
+; VI-NEXT: s_lshr_b32 s56, s24, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 39
+; VI-NEXT: s_lshr_b32 s56, s23, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 40
+; VI-NEXT: s_lshr_b32 s56, s23, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 41
+; VI-NEXT: s_lshr_b32 s56, s23, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 42
+; VI-NEXT: s_lshr_b32 s56, s22, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 43
+; VI-NEXT: s_lshr_b32 s56, s22, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 44
+; VI-NEXT: s_lshr_b32 s56, s21, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 45
+; VI-NEXT: s_lshr_b32 s56, s21, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 46
+; VI-NEXT: s_lshr_b32 s56, s21, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 47
+; VI-NEXT: s_lshr_b32 s56, s20, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 48
+; VI-NEXT: s_lshr_b32 s56, s20, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 49
+; VI-NEXT: s_lshr_b32 s56, s19, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 50
+; VI-NEXT: s_lshr_b32 s56, s19, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 51
+; VI-NEXT: s_lshr_b32 s56, s19, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 52
+; VI-NEXT: s_lshr_b32 s56, s18, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 53
+; VI-NEXT: s_lshr_b32 s56, s18, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 54
+; VI-NEXT: s_lshr_b32 s56, s17, 24
+; VI-NEXT: v_writelane_b32 v33, s56, 55
+; VI-NEXT: s_lshr_b32 s56, s17, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 56
+; VI-NEXT: s_lshr_b32 s56, s17, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 57
+; VI-NEXT: s_lshr_b32 s56, s16, 16
+; VI-NEXT: v_writelane_b32 v33, s56, 58
+; VI-NEXT: s_lshr_b32 s56, s16, 8
+; VI-NEXT: v_writelane_b32 v33, s56, 59
+; VI-NEXT: s_lshr_b64 s[88:89], s[4:5], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 17
+; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 14
+; VI-NEXT: v_writelane_b32 v33, s57, 15
+; VI-NEXT: s_lshr_b64 s[56:57], s[8:9], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 12
+; VI-NEXT: v_writelane_b32 v33, s57, 13
+; VI-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 10
+; VI-NEXT: v_writelane_b32 v33, s57, 11
+; VI-NEXT: s_lshr_b64 s[56:57], s[12:13], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 8
+; VI-NEXT: v_writelane_b32 v33, s57, 9
+; VI-NEXT: s_lshr_b64 s[56:57], s[14:15], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 6
+; VI-NEXT: v_writelane_b32 v33, s57, 7
+; VI-NEXT: s_lshr_b64 s[56:57], s[40:41], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 4
+; VI-NEXT: v_writelane_b32 v33, s57, 5
+; VI-NEXT: s_lshr_b64 s[56:57], s[42:43], 24
+; VI-NEXT: v_writelane_b32 v33, s56, 2
+; VI-NEXT: v_writelane_b32 v33, s57, 3
+; VI-NEXT: s_lshr_b64 s[56:57], s[44:45], 24
; VI-NEXT: v_writelane_b32 v33, s56, 0
-; VI-NEXT: s_lshr_b32 s64, s29, 8
-; VI-NEXT: s_lshr_b32 s50, s28, 8
-; VI-NEXT: s_lshr_b32 s86, s27, 8
-; VI-NEXT: s_lshr_b32 s51, s26, 8
-; VI-NEXT: s_lshr_b32 s66, s25, 8
-; VI-NEXT: s_lshr_b32 s54, s24, 8
-; VI-NEXT: s_lshr_b32 s71, s23, 24
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s52, s23, 8
-; VI-NEXT: s_lshr_b32 s55, s22, 8
-; VI-NEXT: s_lshr_b32 s68, s21, 24
-; VI-NEXT: s_lshr_b32 s65, s21, 8
-; VI-NEXT: s_lshr_b32 s84, s20, 16
-; VI-NEXT: s_lshr_b32 s69, s20, 8
-; VI-NEXT: s_lshr_b32 s80, s19, 24
-; VI-NEXT: s_lshr_b32 s87, s19, 16
-; VI-NEXT: s_lshr_b32 s46, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s47, s18, 8
-; VI-NEXT: s_lshr_b32 s83, s17, 24
-; VI-NEXT: s_lshr_b32 s85, s17, 16
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s53, s16, 16
-; VI-NEXT: s_lshr_b32 s82, s16, 8
+; VI-NEXT: s_lshr_b32 s59, s5, 16
+; VI-NEXT: s_lshr_b32 s70, s5, 8
+; VI-NEXT: s_lshr_b32 s68, s4, 8
+; VI-NEXT: s_lshr_b32 s71, s7, 16
+; VI-NEXT: s_lshr_b32 s81, s7, 8
+; VI-NEXT: s_lshr_b32 s58, s6, 8
+; VI-NEXT: s_lshr_b32 s80, s9, 24
+; VI-NEXT: s_lshr_b32 s84, s9, 16
+; VI-NEXT: s_lshr_b32 s53, s9, 8
+; VI-NEXT: s_lshr_b32 s83, s8, 16
+; VI-NEXT: s_lshr_b32 s61, s8, 8
+; VI-NEXT: s_lshr_b32 s60, s11, 24
+; VI-NEXT: s_lshr_b32 s64, s11, 16
+; VI-NEXT: s_lshr_b32 s63, s11, 8
+; VI-NEXT: s_lshr_b32 s62, s10, 16
+; VI-NEXT: s_lshr_b32 s50, s10, 8
+; VI-NEXT: s_lshr_b32 s73, s13, 24
+; VI-NEXT: s_lshr_b32 s72, s13, 16
+; VI-NEXT: s_lshr_b32 s85, s13, 8
+; VI-NEXT: s_lshr_b32 s51, s12, 16
+; VI-NEXT: s_lshr_b32 s75, s12, 8
+; VI-NEXT: s_lshr_b32 s74, s15, 24
+; VI-NEXT: s_lshr_b32 s66, s15, 16
+; VI-NEXT: s_lshr_b32 s77, s15, 8
+; VI-NEXT: s_lshr_b32 s76, s14, 16
+; VI-NEXT: s_lshr_b32 s54, s14, 8
+; VI-NEXT: s_lshr_b32 s52, s41, 24
+; VI-NEXT: s_lshr_b32 s55, s41, 16
+; VI-NEXT: s_lshr_b32 s65, s41, 8
+; VI-NEXT: s_lshr_b32 s69, s40, 16
+; VI-NEXT: s_lshr_b32 s46, s40, 8
+; VI-NEXT: s_lshr_b32 s78, s43, 24
+; VI-NEXT: s_lshr_b32 s67, s43, 16
+; VI-NEXT: s_lshr_b32 s79, s43, 8
+; VI-NEXT: s_lshr_b32 s82, s42, 16
+; VI-NEXT: s_lshr_b32 s47, s42, 8
+; VI-NEXT: s_lshr_b32 s86, s45, 24
+; VI-NEXT: s_lshr_b32 s87, s45, 16
; VI-NEXT: v_writelane_b32 v33, s57, 1
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; kill: killed $sgpr57
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; kill: killed $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 0
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 2
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 3
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 4
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 5
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 6
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 7
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 8
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 9
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 10
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 11
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 12
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 13
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 14
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 15
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s56, 16
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s57, 17
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s57, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s57, 1
+; VI-NEXT: v_readlane_b32 s48, v33, 18
+; VI-NEXT: s_mov_b32 s49, s59
+; VI-NEXT: v_readlane_b32 s56, v33, 19
+; VI-NEXT: s_mov_b32 s57, s68
+; VI-NEXT: v_readlane_b32 s68, v33, 20
+; VI-NEXT: v_readlane_b32 s59, v33, 21
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -79092,475 +79832,471 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s7, s7, 0
; VI-NEXT: s_add_u32 s4, s4, 3
; VI-NEXT: s_addc_u32 s5, s5, 0
-; VI-NEXT: s_lshr_b32 s46, s5, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s5, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s5, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s4, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s4, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s7, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s7, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 8
-; VI-NEXT: s_lshr_b32 s46, s7, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s6, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s6, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 11
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s9, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 13
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s8, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s11, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s11, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s10, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s10, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s13, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s12, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s15, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s15, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s14, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s14, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s41, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s41, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s41, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s40, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 35
+; VI-NEXT: s_lshr_b32 s88, s45, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 22
+; VI-NEXT: s_lshr_b32 s88, s44, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 23
+; VI-NEXT: s_lshr_b32 s88, s44, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 24
+; VI-NEXT: s_lshr_b32 s88, s29, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 25
+; VI-NEXT: s_lshr_b32 s88, s29, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 26
+; VI-NEXT: s_lshr_b32 s88, s29, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 27
+; VI-NEXT: s_lshr_b32 s88, s28, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 28
+; VI-NEXT: s_lshr_b32 s88, s28, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 29
+; VI-NEXT: s_lshr_b32 s88, s27, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 30
+; VI-NEXT: s_lshr_b32 s88, s27, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 31
+; VI-NEXT: s_lshr_b32 s88, s27, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 32
+; VI-NEXT: s_lshr_b32 s88, s26, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 33
+; VI-NEXT: s_lshr_b32 s88, s26, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 34
+; VI-NEXT: s_lshr_b32 s88, s25, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 35
+; VI-NEXT: s_lshr_b32 s88, s25, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 36
+; VI-NEXT: s_lshr_b32 s88, s25, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 37
+; VI-NEXT: s_lshr_b32 s88, s24, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 38
+; VI-NEXT: s_lshr_b32 s88, s24, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 39
+; VI-NEXT: s_lshr_b32 s88, s23, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 40
+; VI-NEXT: s_lshr_b32 s88, s23, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 41
+; VI-NEXT: s_lshr_b32 s88, s23, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 42
+; VI-NEXT: s_lshr_b32 s88, s22, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 43
+; VI-NEXT: s_lshr_b32 s88, s22, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 44
+; VI-NEXT: s_lshr_b32 s88, s21, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 45
+; VI-NEXT: s_lshr_b32 s88, s21, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 46
+; VI-NEXT: s_lshr_b32 s88, s21, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 47
+; VI-NEXT: s_lshr_b32 s88, s20, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 48
+; VI-NEXT: s_lshr_b32 s88, s20, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 49
+; VI-NEXT: s_lshr_b32 s88, s19, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 50
+; VI-NEXT: s_lshr_b32 s88, s19, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 51
+; VI-NEXT: s_lshr_b32 s88, s19, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 52
+; VI-NEXT: s_lshr_b32 s88, s18, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 53
+; VI-NEXT: s_lshr_b32 s88, s18, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 54
+; VI-NEXT: s_lshr_b32 s88, s17, 24
+; VI-NEXT: v_writelane_b32 v33, s88, 55
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 56
+; VI-NEXT: s_lshr_b32 s88, s17, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 57
+; VI-NEXT: s_lshr_b32 s88, s16, 16
+; VI-NEXT: v_writelane_b32 v33, s88, 58
+; VI-NEXT: s_lshr_b32 s88, s16, 8
+; VI-NEXT: v_writelane_b32 v33, s88, 59
+; VI-NEXT: s_lshr_b64 s[88:89], s[4:5], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 17
+; VI-NEXT: s_lshr_b64 s[88:89], s[6:7], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 14
+; VI-NEXT: v_writelane_b32 v33, s89, 15
+; VI-NEXT: s_lshr_b64 s[88:89], s[8:9], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 12
+; VI-NEXT: v_writelane_b32 v33, s89, 13
+; VI-NEXT: s_lshr_b64 s[88:89], s[10:11], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 10
+; VI-NEXT: v_writelane_b32 v33, s89, 11
+; VI-NEXT: s_lshr_b64 s[88:89], s[12:13], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 8
+; VI-NEXT: v_writelane_b32 v33, s89, 9
+; VI-NEXT: s_lshr_b64 s[88:89], s[14:15], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 6
+; VI-NEXT: v_writelane_b32 v33, s89, 7
+; VI-NEXT: s_lshr_b64 s[88:89], s[40:41], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 4
+; VI-NEXT: v_writelane_b32 v33, s89, 5
+; VI-NEXT: s_lshr_b64 s[88:89], s[42:43], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 2
+; VI-NEXT: v_writelane_b32 v33, s89, 3
+; VI-NEXT: s_lshr_b64 s[88:89], s[44:45], 24
+; VI-NEXT: v_writelane_b32 v33, s88, 0
+; VI-NEXT: s_lshr_b32 s48, s5, 24
+; VI-NEXT: s_lshr_b32 s49, s5, 16
+; VI-NEXT: s_lshr_b32 s70, s5, 8
+; VI-NEXT: s_lshr_b32 s56, s4, 16
+; VI-NEXT: s_lshr_b32 s57, s4, 8
+; VI-NEXT: s_lshr_b32 s68, s7, 24
+; VI-NEXT: s_lshr_b32 s71, s7, 16
+; VI-NEXT: s_lshr_b32 s81, s7, 8
+; VI-NEXT: s_lshr_b32 s59, s6, 16
+; VI-NEXT: s_lshr_b32 s58, s6, 8
+; VI-NEXT: s_lshr_b32 s80, s9, 24
+; VI-NEXT: s_lshr_b32 s84, s9, 16
+; VI-NEXT: s_lshr_b32 s53, s9, 8
+; VI-NEXT: s_lshr_b32 s83, s8, 16
+; VI-NEXT: s_lshr_b32 s61, s8, 8
+; VI-NEXT: s_lshr_b32 s60, s11, 24
+; VI-NEXT: s_lshr_b32 s64, s11, 16
+; VI-NEXT: s_lshr_b32 s63, s11, 8
+; VI-NEXT: s_lshr_b32 s62, s10, 16
+; VI-NEXT: s_lshr_b32 s50, s10, 8
+; VI-NEXT: s_lshr_b32 s73, s13, 24
+; VI-NEXT: s_lshr_b32 s72, s13, 16
+; VI-NEXT: s_lshr_b32 s85, s13, 8
+; VI-NEXT: s_lshr_b32 s51, s12, 16
+; VI-NEXT: s_lshr_b32 s75, s12, 8
+; VI-NEXT: s_lshr_b32 s74, s15, 24
+; VI-NEXT: s_lshr_b32 s66, s15, 16
+; VI-NEXT: s_lshr_b32 s77, s15, 8
+; VI-NEXT: s_lshr_b32 s76, s14, 16
+; VI-NEXT: s_lshr_b32 s54, s14, 8
+; VI-NEXT: s_lshr_b32 s52, s41, 24
+; VI-NEXT: s_lshr_b32 s55, s41, 16
+; VI-NEXT: s_lshr_b32 s65, s41, 8
+; VI-NEXT: s_lshr_b32 s69, s40, 16
; VI-NEXT: s_lshr_b32 s46, s40, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s43, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s43, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s43, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s42, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s42, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s29, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s28, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s27, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s26, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s25, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b64 s[56:57], s[4:5], 24
-; VI-NEXT: v_writelane_b32 v33, s56, 0
-; VI-NEXT: s_lshr_b32 s64, s29, 8
-; VI-NEXT: s_lshr_b32 s50, s28, 8
-; VI-NEXT: s_lshr_b32 s86, s27, 8
-; VI-NEXT: s_lshr_b32 s51, s26, 8
-; VI-NEXT: s_lshr_b32 s66, s25, 8
-; VI-NEXT: s_lshr_b32 s54, s24, 8
-; VI-NEXT: s_lshr_b32 s71, s23, 24
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s52, s23, 8
-; VI-NEXT: s_lshr_b32 s55, s22, 8
-; VI-NEXT: s_lshr_b32 s68, s21, 24
-; VI-NEXT: s_lshr_b32 s65, s21, 8
-; VI-NEXT: s_lshr_b32 s84, s20, 16
-; VI-NEXT: s_lshr_b32 s69, s20, 8
-; VI-NEXT: s_lshr_b32 s80, s19, 24
-; VI-NEXT: s_lshr_b32 s87, s19, 16
-; VI-NEXT: s_lshr_b32 s46, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s47, s18, 8
-; VI-NEXT: s_lshr_b32 s83, s17, 24
-; VI-NEXT: s_lshr_b32 s85, s17, 16
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s53, s16, 16
-; VI-NEXT: s_lshr_b32 s82, s16, 8
-; VI-NEXT: v_writelane_b32 v33, s57, 1
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
+; VI-NEXT: s_lshr_b32 s78, s43, 24
+; VI-NEXT: s_lshr_b32 s67, s43, 16
+; VI-NEXT: s_lshr_b32 s79, s43, 8
+; VI-NEXT: s_lshr_b32 s82, s42, 16
+; VI-NEXT: s_lshr_b32 s47, s42, 8
+; VI-NEXT: s_lshr_b32 s86, s45, 24
+; VI-NEXT: s_lshr_b32 s87, s45, 16
+; VI-NEXT: v_writelane_b32 v33, s89, 1
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: .LBB57_3: ; %end
-; VI-NEXT: v_mov_b32_e32 v9, s48
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: .LBB57_5: ; %end
+; VI-NEXT: v_readlane_b32 s89, v33, 59
+; VI-NEXT: v_mov_b32_e32 v2, s89
+; VI-NEXT: v_readlane_b32 s89, v33, 57
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; VI-NEXT: v_mov_b32_e32 v2, s82
-; VI-NEXT: v_perm_b32 v9, s53, v9, v1
+; VI-NEXT: v_mov_b32_e32 v4, s89
; VI-NEXT: v_perm_b32 v3, s16, v2, v1
-; VI-NEXT: v_mov_b32_e32 v24, s83
+; VI-NEXT: v_readlane_b32 s16, v33, 54
+; VI-NEXT: v_perm_b32 v2, s17, v4, v1
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 52
+; VI-NEXT: v_mov_b32_e32 v6, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 49
+; VI-NEXT: v_perm_b32 v5, s18, v4, v1
+; VI-NEXT: v_perm_b32 v4, s19, v6, v1
+; VI-NEXT: v_mov_b32_e32 v6, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 47
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 44
+; VI-NEXT: v_perm_b32 v7, s20, v6, v1
+; VI-NEXT: v_perm_b32 v6, s21, v8, v1
+; VI-NEXT: v_mov_b32_e32 v8, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 42
+; VI-NEXT: v_mov_b32_e32 v11, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 39
+; VI-NEXT: v_perm_b32 v10, s22, v8, v1
+; VI-NEXT: v_perm_b32 v8, s23, v11, v1
+; VI-NEXT: v_mov_b32_e32 v11, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 37
+; VI-NEXT: v_mov_b32_e32 v14, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 34
+; VI-NEXT: v_perm_b32 v12, s24, v11, v1
+; VI-NEXT: v_perm_b32 v11, s25, v14, v1
+; VI-NEXT: v_mov_b32_e32 v14, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 32
+; VI-NEXT: v_mov_b32_e32 v17, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 29
+; VI-NEXT: v_perm_b32 v15, s26, v14, v1
+; VI-NEXT: v_perm_b32 v14, s27, v17, v1
+; VI-NEXT: v_mov_b32_e32 v17, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 27
+; VI-NEXT: v_mov_b32_e32 v23, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 0
+; VI-NEXT: v_mov_b32_e32 v9, s38
+; VI-NEXT: v_perm_b32 v18, s28, v17, v1
+; VI-NEXT: v_perm_b32 v17, s29, v23, v1
+; VI-NEXT: v_mov_b32_e32 v23, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 58
+; VI-NEXT: v_perm_b32 v9, s16, v9, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 55
+; VI-NEXT: v_mov_b32_e32 v24, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 56
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; VI-NEXT: v_mov_b32_e32 v4, s67
-; VI-NEXT: v_perm_b32 v24, s85, v24, v1
+; VI-NEXT: v_perm_b32 v24, s16, v24, v1
; VI-NEXT: v_or_b32_e32 v3, v3, v9
-; VI-NEXT: v_perm_b32 v2, s17, v4, v1
-; VI-NEXT: v_mov_b32_e32 v13, s38
+; VI-NEXT: v_mov_b32_e32 v13, s36
+; VI-NEXT: v_readlane_b32 s16, v33, 53
; VI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; VI-NEXT: v_mov_b32_e32 v4, s47
-; VI-NEXT: v_perm_b32 v13, s81, v13, v1
+; VI-NEXT: v_mov_b32_e32 v20, vcc_lo
+; VI-NEXT: v_perm_b32 v13, s16, v13, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 50
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 4, v0
-; VI-NEXT: v_perm_b32 v5, s18, v4, v1
-; VI-NEXT: v_mov_b32_e32 v25, s80
+; VI-NEXT: v_mov_b32_e32 v25, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 51
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
-; VI-NEXT: v_mov_b32_e32 v6, s46
-; VI-NEXT: v_perm_b32 v25, s87, v25, v1
+; VI-NEXT: v_perm_b32 v25, s16, v25, v1
; VI-NEXT: v_or_b32_e32 v2, v5, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 8, v0
-; VI-NEXT: v_perm_b32 v4, s19, v6, v1
-; VI-NEXT: v_mov_b32_e32 v17, s36
+; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_readlane_b32 s16, v33, 48
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; VI-NEXT: v_mov_b32_e32 v6, s69
-; VI-NEXT: v_perm_b32 v17, s84, v17, v1
+; VI-NEXT: v_perm_b32 v16, s16, v16, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 45
; VI-NEXT: v_or_b32_e32 v2, v4, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 12, v0
-; VI-NEXT: v_perm_b32 v7, s20, v6, v1
-; VI-NEXT: v_mov_b32_e32 v26, s68
-; VI-NEXT: v_readlane_b32 s16, v33, 57
+; VI-NEXT: v_mov_b32_e32 v26, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 46
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v17
-; VI-NEXT: v_mov_b32_e32 v8, s65
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
; VI-NEXT: v_perm_b32 v26, s16, v26, v1
; VI-NEXT: v_or_b32_e32 v2, v7, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT: v_perm_b32 v6, s21, v8, v1
-; VI-NEXT: v_mov_b32_e32 v19, s34
-; VI-NEXT: v_readlane_b32 s16, v33, 56
+; VI-NEXT: v_mov_b32_e32 v19, s30
+; VI-NEXT: v_readlane_b32 s16, v33, 43
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v26
-; VI-NEXT: v_mov_b32_e32 v8, s55
; VI-NEXT: v_perm_b32 v19, s16, v19, v1
+; VI-NEXT: v_readlane_b32 s16, v33, 40
; VI-NEXT: v_or_b32_e32 v2, v6, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 20, v0
-; VI-NEXT: v_perm_b32 v10, s22, v8, v1
-; VI-NEXT: v_mov_b32_e32 v27, s71
+; VI-NEXT: v_mov_b32_e32 v27, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 41
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v19
-; VI-NEXT: v_mov_b32_e32 v11, s52
-; VI-NEXT: v_perm_b32 v27, s70, v27, v1
+; VI-NEXT: v_perm_b32 v27, s16, v27, v1
; VI-NEXT: v_or_b32_e32 v2, v10, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 24, v0
-; VI-NEXT: v_perm_b32 v8, s23, v11, v1
-; VI-NEXT: v_mov_b32_e32 v20, s30
-; VI-NEXT: v_readlane_b32 s16, v33, 55
+; VI-NEXT: v_readlane_b32 s16, v33, 38
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v27
-; VI-NEXT: v_mov_b32_e32 v11, s54
; VI-NEXT: v_perm_b32 v20, s16, v20, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 53
+; VI-NEXT: v_readlane_b32 s16, v33, 35
; VI-NEXT: v_or_b32_e32 v2, v8, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 28, v0
-; VI-NEXT: v_perm_b32 v12, s24, v11, v1
; VI-NEXT: v_mov_b32_e32 v28, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 54
+; VI-NEXT: v_readlane_b32 s16, v33, 36
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v20
-; VI-NEXT: v_mov_b32_e32 v14, s66
; VI-NEXT: v_perm_b32 v28, s16, v28, v1
; VI-NEXT: v_or_b32_e32 v2, v12, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 32, v0
-; VI-NEXT: v_perm_b32 v11, s25, v14, v1
; VI-NEXT: v_mov_b32_e32 v21, s90
-; VI-NEXT: v_readlane_b32 s16, v33, 52
+; VI-NEXT: v_readlane_b32 s16, v33, 33
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v28
-; VI-NEXT: v_mov_b32_e32 v14, s51
; VI-NEXT: v_perm_b32 v21, s16, v21, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 50
+; VI-NEXT: v_readlane_b32 s16, v33, 30
; VI-NEXT: v_or_b32_e32 v2, v11, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 36, v0
-; VI-NEXT: v_perm_b32 v15, s26, v14, v1
; VI-NEXT: v_mov_b32_e32 v29, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 51
+; VI-NEXT: v_readlane_b32 s16, v33, 31
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v21
-; VI-NEXT: v_mov_b32_e32 v16, s86
; VI-NEXT: v_perm_b32 v29, s16, v29, v1
; VI-NEXT: v_or_b32_e32 v2, v15, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 40, v0
-; VI-NEXT: v_perm_b32 v14, s27, v16, v1
; VI-NEXT: v_mov_b32_e32 v22, s88
-; VI-NEXT: v_readlane_b32 s16, v33, 49
+; VI-NEXT: v_readlane_b32 s16, v33, 28
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
-; VI-NEXT: v_mov_b32_e32 v16, s50
; VI-NEXT: v_perm_b32 v22, s16, v22, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 47
+; VI-NEXT: v_readlane_b32 s16, v33, 25
; VI-NEXT: v_or_b32_e32 v2, v14, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 44, v0
-; VI-NEXT: v_perm_b32 v18, s28, v16, v1
; VI-NEXT: v_mov_b32_e32 v30, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 48
+; VI-NEXT: v_readlane_b32 s16, v33, 26
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; VI-NEXT: v_mov_b32_e32 v23, s64
; VI-NEXT: v_perm_b32 v30, s16, v30, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 46
+; VI-NEXT: v_readlane_b32 s16, v33, 24
; VI-NEXT: v_or_b32_e32 v2, v18, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 48, v0
-; VI-NEXT: v_perm_b32 v16, s29, v23, v1
-; VI-NEXT: v_mov_b32_e32 v23, s78
; VI-NEXT: v_mov_b32_e32 v31, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 45
+; VI-NEXT: v_readlane_b32 s16, v33, 23
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
; VI-NEXT: v_perm_b32 v23, s16, v23, v1
-; VI-NEXT: v_or_b32_e32 v2, v16, v2
+; VI-NEXT: v_or_b32_e32 v2, v17, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 52, v0
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_perm_b32 v2, s44, v31, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v23
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 56, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 44
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
+; VI-NEXT: v_readlane_b32 s16, v33, 22
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 42
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 43
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_perm_b32 v3, s87, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 1
; VI-NEXT: v_perm_b32 v2, s45, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 41
+; VI-NEXT: v_readlane_b32 s16, v33, 2
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_mov_b32_e32 v3, s76
-; VI-NEXT: v_readlane_b32 s16, v33, 40
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s47
+; VI-NEXT: v_perm_b32 v3, s82, v3, v1
; VI-NEXT: v_perm_b32 v2, s42, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 64, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 39
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 37
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 38
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s78
+; VI-NEXT: v_mov_b32_e32 v2, s79
+; VI-NEXT: v_perm_b32 v3, s67, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 3
; VI-NEXT: v_perm_b32 v2, s43, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x44, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 36
+; VI-NEXT: v_readlane_b32 s16, v33, 4
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_mov_b32_e32 v3, s74
-; VI-NEXT: v_readlane_b32 s16, v33, 35
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s46
+; VI-NEXT: v_perm_b32 v3, s69, v3, v1
; VI-NEXT: v_perm_b32 v2, s40, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x48, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 34
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 32
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 33
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s52
+; VI-NEXT: v_mov_b32_e32 v2, s65
+; VI-NEXT: v_perm_b32 v3, s55, v3, v1
+; VI-NEXT: v_readlane_b32 s17, v33, 5
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x4c, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 31
+; VI-NEXT: v_readlane_b32 s16, v33, 6
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_mov_b32_e32 v2, s54
+; VI-NEXT: v_perm_b32 v3, s76, v3, v1
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s72
-; VI-NEXT: v_readlane_b32 s14, v33, 30
-; VI-NEXT: v_perm_b32 v3, s14, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x50, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 29
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 27
-; VI-NEXT: v_mov_b32_e32 v3, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 28
-; VI-NEXT: v_perm_b32 v3, s14, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s74
+; VI-NEXT: v_mov_b32_e32 v2, s77
+; VI-NEXT: v_perm_b32 v3, s66, v3, v1
; VI-NEXT: v_perm_b32 v2, s15, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x54, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 26
+; VI-NEXT: v_readlane_b32 s14, v33, 8
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s14
+; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_perm_b32 v3, s51, v3, v1
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s62
-; VI-NEXT: v_readlane_b32 s12, v33, 25
-; VI-NEXT: v_perm_b32 v3, s12, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x58, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 24
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 22
-; VI-NEXT: v_mov_b32_e32 v3, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 23
-; VI-NEXT: v_perm_b32 v3, s12, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s73
+; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_perm_b32 v3, s72, v3, v1
; VI-NEXT: v_perm_b32 v2, s13, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x5c, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 21
+; VI-NEXT: v_readlane_b32 s12, v33, 10
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s12
+; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_mov_b32_e32 v2, s50
+; VI-NEXT: v_perm_b32 v3, s62, v3, v1
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s60
-; VI-NEXT: v_readlane_b32 s10, v33, 20
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x60, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 19
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 17
-; VI-NEXT: v_mov_b32_e32 v3, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 18
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s60
+; VI-NEXT: v_mov_b32_e32 v2, s63
+; VI-NEXT: v_perm_b32 v3, s64, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x64, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 16
+; VI-NEXT: v_readlane_b32 s10, v33, 12
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: v_mov_b32_e32 v3, s10
+; VI-NEXT: v_mov_b32_e32 v2, s61
+; VI-NEXT: v_perm_b32 v3, s83, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s58
-; VI-NEXT: v_readlane_b32 s8, v33, 15
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x68, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 12
-; VI-NEXT: v_mov_b32_e32 v3, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 13
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v2, s53
+; VI-NEXT: v_perm_b32 v3, s84, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x6c, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 11
+; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: v_mov_b32_e32 v2, s58
+; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s56
-; VI-NEXT: v_readlane_b32 s6, v33, 10
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x70, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 9
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 7
-; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 8
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_perm_b32 v3, s71, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x74, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 6
+; VI-NEXT: v_readlane_b32 s6, v33, 16
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 0
-; VI-NEXT: v_perm_b32 v2, s4, v2, v1
; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_readlane_b32 s4, v33, 5
-; VI-NEXT: v_perm_b32 v3, s4, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s57
+; VI-NEXT: v_perm_b32 v3, s56, v3, v1
+; VI-NEXT: v_perm_b32 v2, s4, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x78, v0
-; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 2
-; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v3, s48
; VI-NEXT: v_perm_b32 v2, s5, v2, v1
-; VI-NEXT: v_perm_b32 v1, s4, v3, v1
+; VI-NEXT: v_perm_b32 v1, s49, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s7, v33, 1
+; VI-NEXT: v_readlane_b32 s17, v33, 7
+; VI-NEXT: v_readlane_b32 s15, v33, 9
+; VI-NEXT: v_readlane_b32 s13, v33, 11
+; VI-NEXT: v_readlane_b32 s11, v33, 13
+; VI-NEXT: v_readlane_b32 s9, v33, 15
+; VI-NEXT: v_readlane_b32 s7, v33, 17
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -79599,162 +80335,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: v_writelane_b32 v33, s70, 0
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: v_writelane_b32 v33, s71, 1
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; kill: killed $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v16i64_to_v128i8_scalar:
; GFX9: ; %bb.0:
@@ -79820,156 +80400,319 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_cmp_lg_u32 s44, 0
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: ; implicit-def: $vgpr30 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 0
-; GFX9-NEXT: s_lshr_b32 s46, s5, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 1
-; GFX9-NEXT: s_lshr_b32 s46, s5, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 2
-; GFX9-NEXT: s_lshr_b32 s46, s4, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 3
-; GFX9-NEXT: s_lshr_b32 s46, s4, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 4
-; GFX9-NEXT: s_lshr_b32 s46, s7, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 5
-; GFX9-NEXT: s_lshr_b32 s46, s7, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 6
-; GFX9-NEXT: s_lshr_b32 s46, s7, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 7
-; GFX9-NEXT: s_lshr_b32 s46, s6, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 8
-; GFX9-NEXT: s_lshr_b32 s46, s6, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 9
-; GFX9-NEXT: s_lshr_b32 s46, s9, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 10
-; GFX9-NEXT: s_lshr_b32 s46, s9, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 11
-; GFX9-NEXT: s_lshr_b32 s46, s9, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 12
-; GFX9-NEXT: s_lshr_b32 s46, s8, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 13
-; GFX9-NEXT: s_lshr_b32 s46, s8, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 14
-; GFX9-NEXT: s_lshr_b32 s46, s11, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 15
-; GFX9-NEXT: s_lshr_b32 s46, s11, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 16
-; GFX9-NEXT: s_lshr_b32 s46, s11, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 17
-; GFX9-NEXT: s_lshr_b32 s46, s10, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 18
-; GFX9-NEXT: s_lshr_b32 s46, s10, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 19
-; GFX9-NEXT: s_lshr_b32 s46, s13, 24
; GFX9-NEXT: v_writelane_b32 v30, s46, 20
-; GFX9-NEXT: s_lshr_b32 s46, s13, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 21
-; GFX9-NEXT: s_lshr_b32 s46, s13, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 22
-; GFX9-NEXT: s_lshr_b32 s46, s12, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 23
-; GFX9-NEXT: s_lshr_b32 s46, s12, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 24
-; GFX9-NEXT: s_lshr_b32 s46, s15, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 25
-; GFX9-NEXT: s_lshr_b32 s46, s15, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 26
-; GFX9-NEXT: s_lshr_b32 s46, s15, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 27
-; GFX9-NEXT: s_lshr_b32 s46, s14, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 28
-; GFX9-NEXT: s_lshr_b32 s46, s14, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 29
-; GFX9-NEXT: s_lshr_b32 s46, s41, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 30
-; GFX9-NEXT: s_lshr_b32 s46, s41, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 31
-; GFX9-NEXT: s_lshr_b32 s46, s41, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 32
-; GFX9-NEXT: s_lshr_b32 s46, s40, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 33
-; GFX9-NEXT: s_lshr_b32 s46, s40, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 34
-; GFX9-NEXT: s_lshr_b32 s46, s43, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 35
-; GFX9-NEXT: s_lshr_b32 s46, s43, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 36
-; GFX9-NEXT: s_lshr_b32 s46, s43, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 37
-; GFX9-NEXT: s_lshr_b32 s46, s42, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 38
-; GFX9-NEXT: s_lshr_b32 s46, s42, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 39
-; GFX9-NEXT: s_lshr_b32 s46, s45, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 40
-; GFX9-NEXT: s_lshr_b32 s46, s45, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 41
-; GFX9-NEXT: s_lshr_b32 s46, s45, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 42
-; GFX9-NEXT: s_lshr_b32 s46, s44, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 43
-; GFX9-NEXT: s_lshr_b32 s46, s44, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 44
-; GFX9-NEXT: s_lshr_b32 s46, s29, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 45
-; GFX9-NEXT: s_lshr_b32 s46, s29, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 46
-; GFX9-NEXT: s_lshr_b32 s46, s29, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 47
-; GFX9-NEXT: s_lshr_b32 s46, s28, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 48
-; GFX9-NEXT: s_lshr_b32 s46, s27, 16
-; GFX9-NEXT: s_lshr_b32 s83, s28, 8
-; GFX9-NEXT: s_lshr_b32 s81, s27, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 49
-; GFX9-NEXT: s_lshr_b32 s53, s27, 8
-; GFX9-NEXT: s_lshr_b32 s82, s26, 16
-; GFX9-NEXT: s_lshr_b32 s67, s26, 8
-; GFX9-NEXT: s_lshr_b32 s84, s25, 24
-; GFX9-NEXT: s_lshr_b32 s96, s25, 16
-; GFX9-NEXT: s_lshr_b32 s65, s25, 8
-; GFX9-NEXT: s_lshr_b32 s86, s24, 16
-; GFX9-NEXT: s_lshr_b32 s55, s24, 8
-; GFX9-NEXT: s_lshr_b32 s98, s23, 24
-; GFX9-NEXT: s_lshr_b32 s87, s23, 16
-; GFX9-NEXT: s_lshr_b32 s66, s23, 8
-; GFX9-NEXT: s_lshr_b32 s97, s22, 16
-; GFX9-NEXT: s_lshr_b32 s64, s22, 8
-; GFX9-NEXT: s_lshr_b32 s49, s21, 24
-; GFX9-NEXT: s_lshr_b32 s99, s21, 16
-; GFX9-NEXT: s_lshr_b32 s80, s21, 8
-; GFX9-NEXT: s_lshr_b32 s39, s20, 16
-; GFX9-NEXT: s_lshr_b32 s70, s20, 8
-; GFX9-NEXT: s_lshr_b32 s48, s19, 24
-; GFX9-NEXT: s_lshr_b32 s38, s19, 16
-; GFX9-NEXT: s_lshr_b32 s68, s19, 8
-; GFX9-NEXT: s_lshr_b32 s54, s18, 16
-; GFX9-NEXT: s_lshr_b32 s71, s18, 8
-; GFX9-NEXT: s_lshr_b32 s52, s17, 24
-; GFX9-NEXT: s_lshr_b32 s50, s17, 16
-; GFX9-NEXT: s_lshr_b32 s69, s17, 8
-; GFX9-NEXT: s_lshr_b32 s51, s16, 16
-; GFX9-NEXT: s_lshr_b32 s85, s16, 8
-; GFX9-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; GFX9-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; GFX9-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; GFX9-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; GFX9-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; GFX9-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; GFX9-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
+; GFX9-NEXT: s_lshr_b32 s47, s11, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 21
+; GFX9-NEXT: s_lshr_b32 s47, s10, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 22
+; GFX9-NEXT: s_lshr_b32 s47, s27, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 23
+; GFX9-NEXT: s_lshr_b32 s47, s26, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 24
+; GFX9-NEXT: s_lshr_b32 s47, s26, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 25
+; GFX9-NEXT: s_lshr_b32 s47, s25, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 26
+; GFX9-NEXT: s_lshr_b32 s47, s25, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 27
+; GFX9-NEXT: s_lshr_b32 s47, s25, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 28
+; GFX9-NEXT: s_lshr_b32 s47, s24, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 29
+; GFX9-NEXT: s_lshr_b32 s47, s24, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 30
+; GFX9-NEXT: s_lshr_b32 s47, s23, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 31
+; GFX9-NEXT: s_lshr_b32 s47, s23, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 32
+; GFX9-NEXT: s_lshr_b32 s47, s23, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 33
+; GFX9-NEXT: s_lshr_b32 s47, s22, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 34
+; GFX9-NEXT: s_lshr_b32 s47, s22, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 35
+; GFX9-NEXT: s_lshr_b32 s47, s21, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 36
+; GFX9-NEXT: s_lshr_b32 s47, s21, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 37
+; GFX9-NEXT: s_lshr_b32 s47, s21, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 38
+; GFX9-NEXT: s_lshr_b32 s47, s20, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 39
+; GFX9-NEXT: s_lshr_b32 s47, s20, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 40
+; GFX9-NEXT: s_lshr_b32 s47, s19, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 41
+; GFX9-NEXT: s_lshr_b32 s47, s19, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 42
+; GFX9-NEXT: s_lshr_b32 s47, s19, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 43
+; GFX9-NEXT: s_lshr_b32 s47, s18, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 44
+; GFX9-NEXT: s_lshr_b32 s47, s18, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 45
+; GFX9-NEXT: s_lshr_b32 s47, s17, 24
+; GFX9-NEXT: v_writelane_b32 v30, s47, 46
+; GFX9-NEXT: s_lshr_b32 s47, s17, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 47
+; GFX9-NEXT: s_lshr_b32 s47, s17, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 48
+; GFX9-NEXT: s_lshr_b32 s47, s16, 16
+; GFX9-NEXT: v_writelane_b32 v30, s47, 49
+; GFX9-NEXT: s_lshr_b32 s47, s16, 8
+; GFX9-NEXT: v_writelane_b32 v30, s47, 50
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[4:5], 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[8:9], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 16
+; GFX9-NEXT: v_writelane_b32 v30, s93, 17
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[10:11], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 14
+; GFX9-NEXT: v_writelane_b32 v30, s93, 15
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 12
+; GFX9-NEXT: v_writelane_b32 v30, s93, 13
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 10
+; GFX9-NEXT: v_writelane_b32 v30, s93, 11
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[40:41], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 8
+; GFX9-NEXT: v_writelane_b32 v30, s93, 9
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[42:43], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 6
+; GFX9-NEXT: v_writelane_b32 v30, s93, 7
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[44:45], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 4
+; GFX9-NEXT: v_writelane_b32 v30, s93, 5
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[28:29], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 2
+; GFX9-NEXT: v_writelane_b32 v30, s93, 3
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[26:27], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 0
+; GFX9-NEXT: s_lshr_b32 s81, s5, 16
+; GFX9-NEXT: s_lshr_b32 s82, s5, 8
+; GFX9-NEXT: s_lshr_b32 s96, s4, 16
+; GFX9-NEXT: s_lshr_b32 s83, s4, 8
+; GFX9-NEXT: s_lshr_b32 s86, s7, 24
+; GFX9-NEXT: s_lshr_b32 s84, s7, 16
+; GFX9-NEXT: s_lshr_b32 s87, s7, 8
+; GFX9-NEXT: s_lshr_b32 s57, s6, 16
+; GFX9-NEXT: s_lshr_b32 s97, s6, 8
+; GFX9-NEXT: s_lshr_b32 s98, s9, 24
+; GFX9-NEXT: s_lshr_b32 s46, s9, 16
+; GFX9-NEXT: s_lshr_b32 s48, s9, 8
+; GFX9-NEXT: s_lshr_b32 s39, s8, 16
+; GFX9-NEXT: s_lshr_b32 s38, s8, 8
+; GFX9-NEXT: s_lshr_b32 s56, s11, 16
+; GFX9-NEXT: s_lshr_b32 s49, s11, 8
+; GFX9-NEXT: s_lshr_b32 s50, s10, 8
+; GFX9-NEXT: s_lshr_b32 s51, s13, 24
+; GFX9-NEXT: s_lshr_b32 s52, s13, 16
+; GFX9-NEXT: s_lshr_b32 s59, s13, 8
+; GFX9-NEXT: s_lshr_b32 s58, s12, 16
+; GFX9-NEXT: s_lshr_b32 s67, s12, 8
+; GFX9-NEXT: s_lshr_b32 s60, s15, 24
+; GFX9-NEXT: s_lshr_b32 s65, s15, 16
+; GFX9-NEXT: s_lshr_b32 s61, s15, 8
+; GFX9-NEXT: s_lshr_b32 s62, s14, 16
+; GFX9-NEXT: s_lshr_b32 s55, s14, 8
+; GFX9-NEXT: s_lshr_b32 s63, s41, 24
+; GFX9-NEXT: s_lshr_b32 s72, s41, 16
+; GFX9-NEXT: s_lshr_b32 s66, s41, 8
+; GFX9-NEXT: s_lshr_b32 s73, s40, 16
+; GFX9-NEXT: s_lshr_b32 s64, s40, 8
+; GFX9-NEXT: s_lshr_b32 s74, s43, 24
+; GFX9-NEXT: s_lshr_b32 s80, s43, 16
+; GFX9-NEXT: s_lshr_b32 s75, s43, 8
+; GFX9-NEXT: s_lshr_b32 s76, s42, 16
+; GFX9-NEXT: s_lshr_b32 s70, s42, 8
+; GFX9-NEXT: s_lshr_b32 s77, s45, 24
+; GFX9-NEXT: s_lshr_b32 s78, s45, 16
+; GFX9-NEXT: s_lshr_b32 s68, s45, 8
+; GFX9-NEXT: s_lshr_b32 s79, s44, 16
+; GFX9-NEXT: s_lshr_b32 s71, s44, 8
+; GFX9-NEXT: s_lshr_b32 s69, s29, 24
+; GFX9-NEXT: s_lshr_b32 s85, s29, 16
+; GFX9-NEXT: s_lshr_b32 s88, s29, 8
+; GFX9-NEXT: s_lshr_b32 s54, s28, 16
+; GFX9-NEXT: s_lshr_b32 s89, s28, 8
+; GFX9-NEXT: s_lshr_b32 s99, s27, 24
+; GFX9-NEXT: s_lshr_b32 s53, s27, 16
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[6:7], 24
+; GFX9-NEXT: v_writelane_b32 v30, s93, 1
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB57_3
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 0
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; kill: killed $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 2
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 3
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 4
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 5
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 6
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 7
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 8
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 9
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 10
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 11
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 12
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 13
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 14
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 15
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 16
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 17
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: v_readlane_b32 s36, v30, 20
+; GFX9-NEXT: s_mov_b32 s37, s81
+; GFX9-NEXT: s_mov_b32 s81, s96
+; GFX9-NEXT: s_mov_b32 s96, s57
+; GFX9-NEXT: s_mov_b32 s47, s48
+; GFX9-NEXT: v_readlane_b32 s57, v30, 21
+; GFX9-NEXT: v_readlane_b32 s48, v30, 22
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -80002,428 +80745,424 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s7, s7, 0
; GFX9-NEXT: s_add_u32 s4, s4, 3
; GFX9-NEXT: s_addc_u32 s5, s5, 0
-; GFX9-NEXT: s_lshr_b32 s46, s5, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 0
-; GFX9-NEXT: s_lshr_b32 s46, s5, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 1
-; GFX9-NEXT: s_lshr_b32 s46, s5, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 2
-; GFX9-NEXT: s_lshr_b32 s46, s4, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 3
-; GFX9-NEXT: s_lshr_b32 s46, s4, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 4
-; GFX9-NEXT: s_lshr_b32 s46, s7, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 5
-; GFX9-NEXT: s_lshr_b32 s46, s7, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 6
-; GFX9-NEXT: s_lshr_b32 s46, s7, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 7
-; GFX9-NEXT: s_lshr_b32 s46, s6, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 8
-; GFX9-NEXT: s_lshr_b32 s46, s6, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 9
-; GFX9-NEXT: s_lshr_b32 s46, s9, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 10
+; GFX9-NEXT: s_lshr_b32 s90, s27, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 23
+; GFX9-NEXT: s_lshr_b32 s90, s26, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 24
+; GFX9-NEXT: s_lshr_b32 s90, s26, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 25
+; GFX9-NEXT: s_lshr_b32 s90, s25, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 26
+; GFX9-NEXT: s_lshr_b32 s90, s25, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 27
+; GFX9-NEXT: s_lshr_b32 s90, s25, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 28
+; GFX9-NEXT: s_lshr_b32 s90, s24, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 29
+; GFX9-NEXT: s_lshr_b32 s90, s24, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 30
+; GFX9-NEXT: s_lshr_b32 s90, s23, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 31
+; GFX9-NEXT: s_lshr_b32 s90, s23, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 32
+; GFX9-NEXT: s_lshr_b32 s90, s23, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 33
+; GFX9-NEXT: s_lshr_b32 s90, s22, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 34
+; GFX9-NEXT: s_lshr_b32 s90, s22, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 35
+; GFX9-NEXT: s_lshr_b32 s90, s21, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 36
+; GFX9-NEXT: s_lshr_b32 s90, s21, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 37
+; GFX9-NEXT: s_lshr_b32 s90, s21, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 38
+; GFX9-NEXT: s_lshr_b32 s90, s20, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 39
+; GFX9-NEXT: s_lshr_b32 s90, s20, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 40
+; GFX9-NEXT: s_lshr_b32 s90, s19, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 41
+; GFX9-NEXT: s_lshr_b32 s90, s19, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 42
+; GFX9-NEXT: s_lshr_b32 s90, s19, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 43
+; GFX9-NEXT: s_lshr_b32 s90, s18, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 44
+; GFX9-NEXT: s_lshr_b32 s90, s18, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 45
+; GFX9-NEXT: s_lshr_b32 s90, s17, 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 46
+; GFX9-NEXT: s_lshr_b32 s90, s17, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 47
+; GFX9-NEXT: s_lshr_b32 s90, s17, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 48
+; GFX9-NEXT: s_lshr_b32 s90, s16, 16
+; GFX9-NEXT: v_writelane_b32 v30, s90, 49
+; GFX9-NEXT: s_lshr_b32 s90, s16, 8
+; GFX9-NEXT: v_writelane_b32 v30, s90, 50
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[4:5], 24
+; GFX9-NEXT: v_writelane_b32 v30, s90, 18
+; GFX9-NEXT: v_writelane_b32 v30, s91, 19
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[8:9], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 16
+; GFX9-NEXT: v_writelane_b32 v30, s93, 17
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[10:11], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 14
+; GFX9-NEXT: v_writelane_b32 v30, s93, 15
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 12
+; GFX9-NEXT: v_writelane_b32 v30, s93, 13
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 10
+; GFX9-NEXT: v_writelane_b32 v30, s93, 11
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[40:41], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 8
+; GFX9-NEXT: v_writelane_b32 v30, s93, 9
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[42:43], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 6
+; GFX9-NEXT: v_writelane_b32 v30, s93, 7
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[44:45], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 4
+; GFX9-NEXT: v_writelane_b32 v30, s93, 5
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[28:29], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 2
+; GFX9-NEXT: v_writelane_b32 v30, s93, 3
+; GFX9-NEXT: s_lshr_b64 s[92:93], s[26:27], 24
+; GFX9-NEXT: v_writelane_b32 v30, s92, 0
+; GFX9-NEXT: s_lshr_b32 s36, s5, 24
+; GFX9-NEXT: s_lshr_b32 s37, s5, 16
+; GFX9-NEXT: s_lshr_b32 s82, s5, 8
+; GFX9-NEXT: s_lshr_b32 s81, s4, 16
+; GFX9-NEXT: s_lshr_b32 s83, s4, 8
+; GFX9-NEXT: s_lshr_b32 s86, s7, 24
+; GFX9-NEXT: s_lshr_b32 s84, s7, 16
+; GFX9-NEXT: s_lshr_b32 s87, s7, 8
+; GFX9-NEXT: s_lshr_b32 s96, s6, 16
+; GFX9-NEXT: s_lshr_b32 s97, s6, 8
+; GFX9-NEXT: s_lshr_b32 s98, s9, 24
; GFX9-NEXT: s_lshr_b32 s46, s9, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 11
-; GFX9-NEXT: s_lshr_b32 s46, s9, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 12
-; GFX9-NEXT: s_lshr_b32 s46, s8, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 13
-; GFX9-NEXT: s_lshr_b32 s46, s8, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 14
-; GFX9-NEXT: s_lshr_b32 s46, s11, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 15
-; GFX9-NEXT: s_lshr_b32 s46, s11, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 16
-; GFX9-NEXT: s_lshr_b32 s46, s11, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 17
-; GFX9-NEXT: s_lshr_b32 s46, s10, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 18
-; GFX9-NEXT: s_lshr_b32 s46, s10, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 19
-; GFX9-NEXT: s_lshr_b32 s46, s13, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 20
-; GFX9-NEXT: s_lshr_b32 s46, s13, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 21
-; GFX9-NEXT: s_lshr_b32 s46, s13, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 22
-; GFX9-NEXT: s_lshr_b32 s46, s12, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 23
-; GFX9-NEXT: s_lshr_b32 s46, s12, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 24
-; GFX9-NEXT: s_lshr_b32 s46, s15, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 25
-; GFX9-NEXT: s_lshr_b32 s46, s15, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 26
-; GFX9-NEXT: s_lshr_b32 s46, s15, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 27
-; GFX9-NEXT: s_lshr_b32 s46, s14, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 28
-; GFX9-NEXT: s_lshr_b32 s46, s14, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 29
-; GFX9-NEXT: s_lshr_b32 s46, s41, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 30
-; GFX9-NEXT: s_lshr_b32 s46, s41, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 31
-; GFX9-NEXT: s_lshr_b32 s46, s41, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 32
-; GFX9-NEXT: s_lshr_b32 s46, s40, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 33
-; GFX9-NEXT: s_lshr_b32 s46, s40, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 34
-; GFX9-NEXT: s_lshr_b32 s46, s43, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 35
-; GFX9-NEXT: s_lshr_b32 s46, s43, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 36
-; GFX9-NEXT: s_lshr_b32 s46, s43, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 37
-; GFX9-NEXT: s_lshr_b32 s46, s42, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 38
-; GFX9-NEXT: s_lshr_b32 s46, s42, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 39
-; GFX9-NEXT: s_lshr_b32 s46, s45, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 40
-; GFX9-NEXT: s_lshr_b32 s46, s45, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 41
-; GFX9-NEXT: s_lshr_b32 s46, s45, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 42
-; GFX9-NEXT: s_lshr_b32 s46, s44, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 43
-; GFX9-NEXT: s_lshr_b32 s46, s44, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 44
-; GFX9-NEXT: s_lshr_b32 s46, s29, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 45
-; GFX9-NEXT: s_lshr_b32 s46, s29, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 46
-; GFX9-NEXT: s_lshr_b32 s46, s29, 8
-; GFX9-NEXT: v_writelane_b32 v30, s46, 47
-; GFX9-NEXT: s_lshr_b32 s46, s28, 16
-; GFX9-NEXT: v_writelane_b32 v30, s46, 48
-; GFX9-NEXT: s_lshr_b32 s46, s27, 16
-; GFX9-NEXT: s_lshr_b32 s83, s28, 8
-; GFX9-NEXT: s_lshr_b32 s81, s27, 24
-; GFX9-NEXT: v_writelane_b32 v30, s46, 49
-; GFX9-NEXT: s_lshr_b32 s53, s27, 8
-; GFX9-NEXT: s_lshr_b32 s82, s26, 16
-; GFX9-NEXT: s_lshr_b32 s67, s26, 8
-; GFX9-NEXT: s_lshr_b32 s84, s25, 24
-; GFX9-NEXT: s_lshr_b32 s96, s25, 16
-; GFX9-NEXT: s_lshr_b32 s65, s25, 8
-; GFX9-NEXT: s_lshr_b32 s86, s24, 16
-; GFX9-NEXT: s_lshr_b32 s55, s24, 8
-; GFX9-NEXT: s_lshr_b32 s98, s23, 24
-; GFX9-NEXT: s_lshr_b32 s87, s23, 16
-; GFX9-NEXT: s_lshr_b32 s66, s23, 8
-; GFX9-NEXT: s_lshr_b32 s97, s22, 16
-; GFX9-NEXT: s_lshr_b32 s64, s22, 8
-; GFX9-NEXT: s_lshr_b32 s49, s21, 24
-; GFX9-NEXT: s_lshr_b32 s99, s21, 16
-; GFX9-NEXT: s_lshr_b32 s80, s21, 8
-; GFX9-NEXT: s_lshr_b32 s39, s20, 16
-; GFX9-NEXT: s_lshr_b32 s70, s20, 8
-; GFX9-NEXT: s_lshr_b32 s48, s19, 24
-; GFX9-NEXT: s_lshr_b32 s38, s19, 16
-; GFX9-NEXT: s_lshr_b32 s68, s19, 8
-; GFX9-NEXT: s_lshr_b32 s54, s18, 16
-; GFX9-NEXT: s_lshr_b32 s71, s18, 8
-; GFX9-NEXT: s_lshr_b32 s52, s17, 24
-; GFX9-NEXT: s_lshr_b32 s50, s17, 16
-; GFX9-NEXT: s_lshr_b32 s69, s17, 8
-; GFX9-NEXT: s_lshr_b32 s51, s16, 16
-; GFX9-NEXT: s_lshr_b32 s85, s16, 8
-; GFX9-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; GFX9-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; GFX9-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; GFX9-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; GFX9-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; GFX9-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; GFX9-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
+; GFX9-NEXT: s_lshr_b32 s47, s9, 8
+; GFX9-NEXT: s_lshr_b32 s39, s8, 16
+; GFX9-NEXT: s_lshr_b32 s38, s8, 8
+; GFX9-NEXT: s_lshr_b32 s57, s11, 24
+; GFX9-NEXT: s_lshr_b32 s56, s11, 16
+; GFX9-NEXT: s_lshr_b32 s49, s11, 8
+; GFX9-NEXT: s_lshr_b32 s48, s10, 16
+; GFX9-NEXT: s_lshr_b32 s50, s10, 8
+; GFX9-NEXT: s_lshr_b32 s51, s13, 24
+; GFX9-NEXT: s_lshr_b32 s52, s13, 16
+; GFX9-NEXT: s_lshr_b32 s59, s13, 8
+; GFX9-NEXT: s_lshr_b32 s58, s12, 16
+; GFX9-NEXT: s_lshr_b32 s67, s12, 8
+; GFX9-NEXT: s_lshr_b32 s60, s15, 24
+; GFX9-NEXT: s_lshr_b32 s65, s15, 16
+; GFX9-NEXT: s_lshr_b32 s61, s15, 8
+; GFX9-NEXT: s_lshr_b32 s62, s14, 16
+; GFX9-NEXT: s_lshr_b32 s55, s14, 8
+; GFX9-NEXT: s_lshr_b32 s63, s41, 24
+; GFX9-NEXT: s_lshr_b32 s72, s41, 16
+; GFX9-NEXT: s_lshr_b32 s66, s41, 8
+; GFX9-NEXT: s_lshr_b32 s73, s40, 16
+; GFX9-NEXT: s_lshr_b32 s64, s40, 8
+; GFX9-NEXT: s_lshr_b32 s74, s43, 24
+; GFX9-NEXT: s_lshr_b32 s80, s43, 16
+; GFX9-NEXT: s_lshr_b32 s75, s43, 8
+; GFX9-NEXT: s_lshr_b32 s76, s42, 16
+; GFX9-NEXT: s_lshr_b32 s70, s42, 8
+; GFX9-NEXT: s_lshr_b32 s77, s45, 24
+; GFX9-NEXT: s_lshr_b32 s78, s45, 16
+; GFX9-NEXT: s_lshr_b32 s68, s45, 8
+; GFX9-NEXT: s_lshr_b32 s79, s44, 16
+; GFX9-NEXT: s_lshr_b32 s71, s44, 8
+; GFX9-NEXT: s_lshr_b32 s69, s29, 24
+; GFX9-NEXT: s_lshr_b32 s85, s29, 16
+; GFX9-NEXT: s_lshr_b32 s88, s29, 8
+; GFX9-NEXT: s_lshr_b32 s54, s28, 16
+; GFX9-NEXT: s_lshr_b32 s89, s28, 8
+; GFX9-NEXT: s_lshr_b32 s99, s27, 24
+; GFX9-NEXT: s_lshr_b32 s53, s27, 16
+; GFX9-NEXT: s_lshr_b64 s[90:91], s[6:7], 24
+; GFX9-NEXT: v_writelane_b32 v30, s93, 1
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: .LBB57_3: ; %end
-; GFX9-NEXT: v_mov_b32_e32 v7, s36
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: v_readlane_b32 s91, v30, 50
+; GFX9-NEXT: v_mov_b32_e32 v2, s91
+; GFX9-NEXT: v_readlane_b32 s91, v30, 48
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v2, s85
-; GFX9-NEXT: v_perm_b32 v7, s51, v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, s91
; GFX9-NEXT: v_perm_b32 v3, s16, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v21, s52
-; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX9-NEXT: v_mov_b32_e32 v4, s69
-; GFX9-NEXT: v_perm_b32 v21, s50, v21, v1
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-NEXT: v_readlane_b32 s16, v30, 45
; GFX9-NEXT: v_perm_b32 v2, s17, v4, v1
-; GFX9-NEXT: v_mov_b32_e32 v11, s34
+; GFX9-NEXT: v_mov_b32_e32 v4, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 43
+; GFX9-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 40
+; GFX9-NEXT: v_perm_b32 v5, s18, v4, v1
+; GFX9-NEXT: v_perm_b32 v4, s19, v6, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 38
+; GFX9-NEXT: v_mov_b32_e32 v9, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 35
+; GFX9-NEXT: v_perm_b32 v7, s20, v6, v1
+; GFX9-NEXT: v_perm_b32 v6, s21, v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v9, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 33
+; GFX9-NEXT: v_mov_b32_e32 v12, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 30
+; GFX9-NEXT: v_perm_b32 v10, s22, v9, v1
+; GFX9-NEXT: v_perm_b32 v9, s23, v12, v1
+; GFX9-NEXT: v_mov_b32_e32 v12, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 28
+; GFX9-NEXT: v_mov_b32_e32 v15, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 0
+; GFX9-NEXT: v_mov_b32_e32 v19, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 25
+; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
+; GFX9-NEXT: v_perm_b32 v12, s25, v15, v1
+; GFX9-NEXT: v_readlane_b32 s17, v30, 1
+; GFX9-NEXT: v_mov_b32_e32 v15, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 23
+; GFX9-NEXT: v_mov_b32_e32 v20, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 2
+; GFX9-NEXT: v_mov_b32_e32 v8, s34
+; GFX9-NEXT: v_perm_b32 v16, s26, v15, v1
+; GFX9-NEXT: v_perm_b32 v15, s27, v20, v1
+; GFX9-NEXT: v_mov_b32_e32 v20, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 49
+; GFX9-NEXT: v_perm_b32 v8, s16, v8, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 46
+; GFX9-NEXT: v_mov_b32_e32 v21, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 47
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX9-NEXT: v_perm_b32 v21, s16, v21, v1
+; GFX9-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX9-NEXT: v_mov_b32_e32 v11, s30
+; GFX9-NEXT: v_readlane_b32 s16, v30, 44
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v21
-; GFX9-NEXT: v_mov_b32_e32 v4, s71
-; GFX9-NEXT: v_perm_b32 v11, s54, v11, v1
+; GFX9-NEXT: v_perm_b32 v11, s16, v11, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 41
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_perm_b32 v5, s18, v4, v1
-; GFX9-NEXT: v_mov_b32_e32 v22, s48
+; GFX9-NEXT: v_mov_b32_e32 v22, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 42
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v11
-; GFX9-NEXT: v_mov_b32_e32 v6, s68
-; GFX9-NEXT: v_perm_b32 v22, s38, v22, v1
+; GFX9-NEXT: v_perm_b32 v22, s16, v22, v1
; GFX9-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX9-NEXT: v_perm_b32 v4, s19, v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v14, s30
+; GFX9-NEXT: v_mov_b32_e32 v14, vcc_lo
+; GFX9-NEXT: v_readlane_b32 s16, v30, 39
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; GFX9-NEXT: v_mov_b32_e32 v6, s70
-; GFX9-NEXT: v_perm_b32 v14, s39, v14, v1
+; GFX9-NEXT: v_perm_b32 v14, s16, v14, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 36
; GFX9-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX9-NEXT: v_perm_b32 v8, s20, v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v23, s49
+; GFX9-NEXT: v_mov_b32_e32 v23, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 37
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:12
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v14
-; GFX9-NEXT: v_mov_b32_e32 v9, s80
-; GFX9-NEXT: v_perm_b32 v23, s99, v23, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v8, v2
-; GFX9-NEXT: v_perm_b32 v6, s21, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v15, s94
+; GFX9-NEXT: v_perm_b32 v23, s16, v23, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX9-NEXT: v_mov_b32_e32 v17, s94
+; GFX9-NEXT: v_readlane_b32 s16, v30, 34
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v23
-; GFX9-NEXT: v_mov_b32_e32 v9, s64
-; GFX9-NEXT: v_perm_b32 v15, s97, v15, v1
+; GFX9-NEXT: v_perm_b32 v17, s16, v17, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 31
; GFX9-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX9-NEXT: v_perm_b32 v10, s22, v9, v1
-; GFX9-NEXT: v_mov_b32_e32 v24, s98
+; GFX9-NEXT: v_mov_b32_e32 v24, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 32
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v15
-; GFX9-NEXT: v_mov_b32_e32 v12, s66
-; GFX9-NEXT: v_perm_b32 v24, s87, v24, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX9-NEXT: v_perm_b32 v24, s16, v24, v1
; GFX9-NEXT: v_or_b32_e32 v2, v10, v2
-; GFX9-NEXT: v_perm_b32 v9, s23, v12, v1
-; GFX9-NEXT: v_mov_b32_e32 v16, s92
+; GFX9-NEXT: v_mov_b32_e32 v18, s92
+; GFX9-NEXT: v_readlane_b32 s16, v30, 29
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v24
-; GFX9-NEXT: v_mov_b32_e32 v12, s55
-; GFX9-NEXT: v_perm_b32 v16, s86, v16, v1
+; GFX9-NEXT: v_perm_b32 v18, s16, v18, v1
+; GFX9-NEXT: v_readlane_b32 s16, v30, 26
; GFX9-NEXT: v_or_b32_e32 v2, v9, v2
-; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
-; GFX9-NEXT: v_mov_b32_e32 v25, s84
+; GFX9-NEXT: v_mov_b32_e32 v25, s16
+; GFX9-NEXT: v_readlane_b32 s16, v30, 27
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v16
-; GFX9-NEXT: v_mov_b32_e32 v17, s65
-; GFX9-NEXT: v_perm_b32 v25, s96, v25, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v18
+; GFX9-NEXT: v_perm_b32 v25, s16, v25, v1
; GFX9-NEXT: v_or_b32_e32 v2, v13, v2
-; GFX9-NEXT: v_perm_b32 v12, s25, v17, v1
-; GFX9-NEXT: v_mov_b32_e32 v17, s90
+; GFX9-NEXT: v_readlane_b32 s16, v30, 24
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:32
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; GFX9-NEXT: v_mov_b32_e32 v18, s67
-; GFX9-NEXT: v_perm_b32 v17, s82, v17, v1
+; GFX9-NEXT: v_perm_b32 v19, s16, v19, v1
; GFX9-NEXT: v_or_b32_e32 v2, v12, v2
-; GFX9-NEXT: v_perm_b32 v18, s26, v18, v1
-; GFX9-NEXT: v_mov_b32_e32 v26, s81
-; GFX9-NEXT: v_readlane_b32 s16, v30, 49
+; GFX9-NEXT: v_mov_b32_e32 v26, s99
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v17
-; GFX9-NEXT: v_mov_b32_e32 v19, s53
-; GFX9-NEXT: v_perm_b32 v26, s16, v26, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v18, v2
-; GFX9-NEXT: v_perm_b32 v19, s27, v19, v1
-; GFX9-NEXT: v_mov_b32_e32 v20, s88
-; GFX9-NEXT: v_readlane_b32 s16, v30, 48
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v19
+; GFX9-NEXT: v_perm_b32 v26, s53, v26, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v16, v2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v26
-; GFX9-NEXT: v_mov_b32_e32 v27, s83
-; GFX9-NEXT: v_perm_b32 v20, s16, v20, v1
-; GFX9-NEXT: v_readlane_b32 s16, v30, 47
-; GFX9-NEXT: v_or_b32_e32 v2, v19, v2
-; GFX9-NEXT: v_mov_b32_e32 v28, s16
+; GFX9-NEXT: v_mov_b32_e32 v27, s89
+; GFX9-NEXT: v_perm_b32 v20, s54, v20, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v15, v2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: v_perm_b32 v2, s28, v27, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v20
-; GFX9-NEXT: v_readlane_b32 s16, v30, 45
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 46
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, s69
+; GFX9-NEXT: v_readlane_b32 s17, v30, 3
+; GFX9-NEXT: v_mov_b32_e32 v28, s88
+; GFX9-NEXT: v_perm_b32 v3, s85, v3, v1
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
; GFX9-NEXT: v_perm_b32 v2, s29, v28, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 4
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 44
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s78
-; GFX9-NEXT: v_readlane_b32 s16, v30, 43
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s71
+; GFX9-NEXT: v_perm_b32 v3, s79, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s44, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 42
+; GFX9-NEXT: v_mov_b32_e32 v3, s77
+; GFX9-NEXT: v_readlane_b32 s17, v30, 5
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 40
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 41
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s68
+; GFX9-NEXT: v_perm_b32 v3, s78, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s45, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 6
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 39
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:60
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s76
-; GFX9-NEXT: v_readlane_b32 s16, v30, 38
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s70
+; GFX9-NEXT: v_perm_b32 v3, s76, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s42, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 37
+; GFX9-NEXT: v_mov_b32_e32 v3, s74
+; GFX9-NEXT: v_readlane_b32 s17, v30, 7
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 35
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 36
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s75
+; GFX9-NEXT: v_perm_b32 v3, s80, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s43, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 8
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 34
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_mov_b32_e32 v3, s74
-; GFX9-NEXT: v_readlane_b32 s16, v30, 33
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s64
+; GFX9-NEXT: v_perm_b32 v3, s73, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s40, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 32
+; GFX9-NEXT: v_mov_b32_e32 v3, s63
+; GFX9-NEXT: v_readlane_b32 s17, v30, 9
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 30
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
-; GFX9-NEXT: v_readlane_b32 s16, v30, 31
-; GFX9-NEXT: v_perm_b32 v3, s16, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s66
+; GFX9-NEXT: v_perm_b32 v3, s72, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s41, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s16, v30, 10
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 29
+; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
-; GFX9-NEXT: v_mov_b32_e32 v2, s16
+; GFX9-NEXT: v_mov_b32_e32 v2, s55
+; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s14, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s72
-; GFX9-NEXT: v_readlane_b32 s14, v30, 28
-; GFX9-NEXT: v_perm_b32 v3, s14, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 27
+; GFX9-NEXT: v_mov_b32_e32 v3, s60
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:80
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
-; GFX9-NEXT: v_readlane_b32 s14, v30, 25
-; GFX9-NEXT: v_mov_b32_e32 v3, s14
-; GFX9-NEXT: v_readlane_b32 s14, v30, 26
-; GFX9-NEXT: v_perm_b32 v3, s14, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s61
+; GFX9-NEXT: v_perm_b32 v3, s65, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s15, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s14, v30, 12
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 24
+; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
-; GFX9-NEXT: v_mov_b32_e32 v2, s14
+; GFX9-NEXT: v_mov_b32_e32 v2, s67
+; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s12, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s62
-; GFX9-NEXT: v_readlane_b32 s12, v30, 23
-; GFX9-NEXT: v_perm_b32 v3, s12, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 22
+; GFX9-NEXT: v_mov_b32_e32 v3, s51
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:88
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
-; GFX9-NEXT: v_readlane_b32 s12, v30, 20
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
-; GFX9-NEXT: v_readlane_b32 s12, v30, 21
-; GFX9-NEXT: v_perm_b32 v3, s12, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s59
+; GFX9-NEXT: v_perm_b32 v3, s52, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s13, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s12, v30, 14
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 19
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
-; GFX9-NEXT: v_mov_b32_e32 v2, s12
+; GFX9-NEXT: v_mov_b32_e32 v2, s50
+; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s10, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s60
-; GFX9-NEXT: v_readlane_b32 s10, v30, 18
-; GFX9-NEXT: v_perm_b32 v3, s10, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 17
+; GFX9-NEXT: v_mov_b32_e32 v3, s57
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:96
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_readlane_b32 s10, v30, 15
-; GFX9-NEXT: v_mov_b32_e32 v3, s10
-; GFX9-NEXT: v_readlane_b32 s10, v30, 16
-; GFX9-NEXT: v_perm_b32 v3, s10, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s49
+; GFX9-NEXT: v_perm_b32 v3, s56, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s11, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s10, v30, 16
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 14
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v2, s38
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s8, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s58
-; GFX9-NEXT: v_readlane_b32 s8, v30, 13
-; GFX9-NEXT: v_perm_b32 v3, s8, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s8, v30, 12
+; GFX9-NEXT: v_mov_b32_e32 v3, s98
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:104
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
-; GFX9-NEXT: v_readlane_b32 s8, v30, 10
-; GFX9-NEXT: v_mov_b32_e32 v3, s8
-; GFX9-NEXT: v_readlane_b32 s8, v30, 11
-; GFX9-NEXT: v_perm_b32 v3, s8, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s47
+; GFX9-NEXT: v_perm_b32 v3, s46, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s9, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s8, v30, 9
+; GFX9-NEXT: v_mov_b32_e32 v3, s90
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
-; GFX9-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-NEXT: v_mov_b32_e32 v2, s97
+; GFX9-NEXT: v_perm_b32 v3, s96, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s6, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s56
-; GFX9-NEXT: v_readlane_b32 s6, v30, 8
-; GFX9-NEXT: v_perm_b32 v3, s6, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 7
+; GFX9-NEXT: v_mov_b32_e32 v3, s86
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:112
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_readlane_b32 s6, v30, 5
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: v_readlane_b32 s6, v30, 6
-; GFX9-NEXT: v_perm_b32 v3, s6, v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s87
+; GFX9-NEXT: v_perm_b32 v3, s84, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s7, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_readlane_b32 s6, v30, 18
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 4
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s83
+; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s46
-; GFX9-NEXT: v_readlane_b32 s4, v30, 3
-; GFX9-NEXT: v_perm_b32 v3, s4, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_readlane_b32 s4, v30, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: v_readlane_b32 s4, v30, 1
+; GFX9-NEXT: v_mov_b32_e32 v2, s82
+; GFX9-NEXT: v_mov_b32_e32 v3, s36
; GFX9-NEXT: v_perm_b32 v2, s5, v2, v1
-; GFX9-NEXT: v_perm_b32 v1, s4, v3, v1
+; GFX9-NEXT: v_perm_b32 v1, s37, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
; GFX9-NEXT: v_readlane_b32 s30, v29, 34
+; GFX9-NEXT: v_readlane_b32 s17, v30, 11
+; GFX9-NEXT: v_readlane_b32 s15, v30, 13
+; GFX9-NEXT: v_readlane_b32 s13, v30, 15
+; GFX9-NEXT: v_readlane_b32 s11, v30, 17
+; GFX9-NEXT: v_readlane_b32 s7, v30, 19
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s31, v29, 35
; GFX9-NEXT: v_readlane_b32 s99, v29, 33
@@ -80466,154 +81205,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB57_2
;
; GFX11-LABEL: bitcast_v16i64_to_v128i8_scalar:
; GFX11: ; %bb.0:
@@ -80682,161 +81273,302 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s103, 0
; GFX11-NEXT: ; implicit-def: $vgpr26 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr25 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB57_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s42, s5, 24
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s27, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s5, 16
-; GFX11-NEXT: s_lshr_b32 s34, s27, 16
-; GFX11-NEXT: s_lshr_b32 s35, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s26, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s5, 8
-; GFX11-NEXT: s_lshr_b32 s36, s26, 8
-; GFX11-NEXT: s_lshr_b32 s39, s25, 24
-; GFX11-NEXT: s_lshr_b32 s48, s25, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 2
+; GFX11-NEXT: s_lshr_b32 s43, s8, 16
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[4:5], 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 18
+; GFX11-NEXT: s_lshr_b32 s43, s12, 16
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[8:9], 24
+; GFX11-NEXT: s_lshr_b32 s31, s5, 24
+; GFX11-NEXT: s_lshr_b32 s104, s5, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 19
+; GFX11-NEXT: s_lshr_b32 s43, s22, 16
+; GFX11-NEXT: s_lshr_b32 s30, s5, 8
; GFX11-NEXT: s_lshr_b32 s42, s4, 16
-; GFX11-NEXT: s_lshr_b32 s38, s25, 8
-; GFX11-NEXT: s_lshr_b32 s50, s24, 16
-; GFX11-NEXT: s_lshr_b32 s49, s24, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s4, 8
-; GFX11-NEXT: s_lshr_b32 s52, s23, 24
-; GFX11-NEXT: s_lshr_b32 s53, s23, 16
-; GFX11-NEXT: s_lshr_b32 s51, s23, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s7, 24
-; GFX11-NEXT: s_lshr_b32 s69, s22, 16
-; GFX11-NEXT: s_lshr_b32 s54, s22, 8
-; GFX11-NEXT: s_lshr_b32 s55, s21, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s7, 16
-; GFX11-NEXT: s_lshr_b32 s64, s21, 16
-; GFX11-NEXT: s_lshr_b32 s65, s21, 8
-; GFX11-NEXT: s_lshr_b32 s82, s20, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s7, 8
-; GFX11-NEXT: s_lshr_b32 s68, s20, 8
-; GFX11-NEXT: s_lshr_b32 s66, s19, 24
-; GFX11-NEXT: s_lshr_b32 s67, s19, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 7
-; GFX11-NEXT: s_lshr_b32 s42, s6, 16
-; GFX11-NEXT: s_lshr_b32 s85, s19, 8
-; GFX11-NEXT: s_lshr_b32 s86, s18, 16
-; GFX11-NEXT: s_lshr_b32 s87, s18, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 8
-; GFX11-NEXT: s_lshr_b32 s42, s6, 8
-; GFX11-NEXT: s_lshr_b32 s70, s17, 24
-; GFX11-NEXT: s_lshr_b32 s71, s17, 16
-; GFX11-NEXT: s_lshr_b32 s96, s17, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 9
-; GFX11-NEXT: s_lshr_b32 s42, s9, 24
-; GFX11-NEXT: s_lshr_b32 s97, s16, 16
-; GFX11-NEXT: s_lshr_b32 s98, s16, 8
-; GFX11-NEXT: s_lshr_b32 s80, s3, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 10
-; GFX11-NEXT: s_lshr_b32 s42, s9, 16
-; GFX11-NEXT: s_lshr_b32 s81, s3, 16
-; GFX11-NEXT: s_lshr_b32 s99, s3, 8
-; GFX11-NEXT: s_lshr_b32 s100, s2, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 11
-; GFX11-NEXT: s_lshr_b32 s42, s9, 8
-; GFX11-NEXT: s_lshr_b32 s101, s2, 8
-; GFX11-NEXT: s_lshr_b32 s83, s1, 24
-; GFX11-NEXT: s_lshr_b32 s84, s1, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 12
-; GFX11-NEXT: s_lshr_b32 s42, s8, 16
-; GFX11-NEXT: s_lshr_b32 s102, s1, 8
-; GFX11-NEXT: s_lshr_b32 s103, s0, 16
-; GFX11-NEXT: s_lshr_b32 s104, s0, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 13
-; GFX11-NEXT: s_lshr_b32 s42, s8, 8
-; GFX11-NEXT: s_lshr_b64 s[44:45], s[6:7], 24
-; GFX11-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; GFX11-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 14
-; GFX11-NEXT: s_lshr_b32 s42, s11, 24
-; GFX11-NEXT: s_lshr_b64 s[58:59], s[12:13], 24
-; GFX11-NEXT: s_lshr_b64 s[60:61], s[14:15], 24
-; GFX11-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 15
-; GFX11-NEXT: s_lshr_b32 s42, s11, 16
-; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
-; GFX11-NEXT: s_lshr_b64 s[76:77], s[26:27], 24
-; GFX11-NEXT: s_lshr_b64 s[72:73], s[24:25], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 16
-; GFX11-NEXT: s_lshr_b32 s42, s11, 8
-; GFX11-NEXT: s_lshr_b64 s[78:79], s[22:23], 24
+; GFX11-NEXT: s_lshr_b32 s34, s4, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 20
+; GFX11-NEXT: s_lshr_b32 s43, s20, 16
+; GFX11-NEXT: s_lshr_b32 s35, s7, 24
+; GFX11-NEXT: s_lshr_b32 s37, s7, 16
+; GFX11-NEXT: s_lshr_b32 s36, s7, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 21
+; GFX11-NEXT: s_lshr_b32 s43, s20, 8
+; GFX11-NEXT: s_lshr_b32 s61, s6, 16
+; GFX11-NEXT: s_lshr_b32 s38, s6, 8
+; GFX11-NEXT: s_lshr_b32 s48, s9, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 22
+; GFX11-NEXT: s_lshr_b32 s43, s19, 8
+; GFX11-NEXT: s_lshr_b32 s39, s9, 16
+; GFX11-NEXT: s_lshr_b32 s45, s9, 8
+; GFX11-NEXT: s_lshr_b32 s44, s8, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 23
+; GFX11-NEXT: s_lshr_b32 s43, s18, 16
+; GFX11-NEXT: s_lshr_b32 s47, s11, 24
+; GFX11-NEXT: s_lshr_b32 s49, s11, 16
+; GFX11-NEXT: s_lshr_b32 s46, s11, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 24
+; GFX11-NEXT: s_lshr_b32 s43, s18, 8
+; GFX11-NEXT: s_lshr_b32 s51, s10, 16
+; GFX11-NEXT: s_lshr_b32 s50, s10, 8
+; GFX11-NEXT: s_lshr_b32 s57, s13, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 25
+; GFX11-NEXT: s_lshr_b32 s43, s17, 24
+; GFX11-NEXT: s_lshr_b32 s56, s13, 16
+; GFX11-NEXT: s_lshr_b32 s59, s13, 8
+; GFX11-NEXT: s_lshr_b32 s58, s12, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 26
+; GFX11-NEXT: s_lshr_b32 s43, s17, 16
+; GFX11-NEXT: s_lshr_b32 s60, s15, 24
+; GFX11-NEXT: s_lshr_b32 s53, s15, 16
+; GFX11-NEXT: s_lshr_b32 s52, s15, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 27
+; GFX11-NEXT: s_lshr_b32 s43, s17, 8
+; GFX11-NEXT: s_lshr_b32 s62, s14, 16
+; GFX11-NEXT: s_lshr_b32 s63, s14, 8
+; GFX11-NEXT: s_lshr_b32 s54, s41, 24
+; GFX11-NEXT: v_writelane_b32 v26, s43, 28
+; GFX11-NEXT: s_lshr_b32 s43, s16, 16
+; GFX11-NEXT: s_lshr_b32 s55, s41, 16
+; GFX11-NEXT: s_lshr_b32 s64, s41, 8
+; GFX11-NEXT: s_lshr_b32 s66, s40, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 29
+; GFX11-NEXT: s_lshr_b32 s43, s16, 8
+; GFX11-NEXT: s_lshr_b32 s65, s40, 8
+; GFX11-NEXT: s_lshr_b32 s69, s29, 24
+; GFX11-NEXT: s_lshr_b32 s71, s29, 16
+; GFX11-NEXT: v_writelane_b32 v26, s43, 30
+; GFX11-NEXT: s_lshr_b32 s43, s3, 24
+; GFX11-NEXT: s_lshr_b32 s67, s29, 8
+; GFX11-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-NEXT: s_lshr_b32 s80, s28, 8
+; GFX11-NEXT: v_writelane_b32 v26, s43, 31
+; GFX11-NEXT: s_lshr_b32 s43, s3, 16
+; GFX11-NEXT: s_lshr_b32 s82, s27, 24
+; GFX11-NEXT: v_writelane_b32 v25, s43, 0
+; GFX11-NEXT: s_lshr_b32 s43, s3, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
+; GFX11-NEXT: s_lshr_b32 s83, s27, 16
+; GFX11-NEXT: s_lshr_b32 s84, s27, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 1
+; GFX11-NEXT: s_lshr_b32 s43, s2, 16
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: s_lshr_b32 s86, s26, 16
+; GFX11-NEXT: s_lshr_b32 s85, s26, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 2
+; GFX11-NEXT: s_lshr_b32 s43, s2, 8
+; GFX11-NEXT: v_writelane_b32 v26, s74, 14
+; GFX11-NEXT: s_lshr_b32 s96, s25, 24
+; GFX11-NEXT: s_lshr_b32 s97, s25, 16
+; GFX11-NEXT: v_writelane_b32 v25, s43, 3
+; GFX11-NEXT: s_lshr_b32 s43, s1, 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 15
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[10:11], 24
+; GFX11-NEXT: s_lshr_b32 s87, s25, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 4
+; GFX11-NEXT: s_lshr_b32 s43, s1, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 12
+; GFX11-NEXT: s_lshr_b32 s99, s24, 16
+; GFX11-NEXT: s_lshr_b32 s98, s24, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 5
+; GFX11-NEXT: s_lshr_b32 s43, s1, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 13
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[12:13], 24
+; GFX11-NEXT: s_lshr_b32 s101, s23, 24
+; GFX11-NEXT: v_writelane_b32 v25, s43, 6
+; GFX11-NEXT: s_lshr_b32 s43, s0, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 10
+; GFX11-NEXT: s_lshr_b32 s102, s23, 16
+; GFX11-NEXT: s_lshr_b32 s100, s23, 8
+; GFX11-NEXT: v_writelane_b32 v25, s43, 7
+; GFX11-NEXT: s_lshr_b32 s43, s0, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 11
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[14:15], 24
+; GFX11-NEXT: s_lshr_b32 s76, s22, 8
+; GFX11-NEXT: s_lshr_b32 s78, s21, 24
+; GFX11-NEXT: s_lshr_b32 s77, s21, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 8
+; GFX11-NEXT: s_lshr_b32 s68, s21, 8
+; GFX11-NEXT: s_lshr_b32 s79, s19, 24
+; GFX11-NEXT: s_lshr_b32 s70, s19, 16
+; GFX11-NEXT: v_writelane_b32 v25, s43, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 9
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[6:7], 24
; GFX11-NEXT: s_lshr_b64 s[88:89], s[20:21], 24
; GFX11-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 17
-; GFX11-NEXT: s_lshr_b32 s42, s10, 16
+; GFX11-NEXT: v_writelane_b32 v26, s74, 6
; GFX11-NEXT: s_lshr_b64 s[92:93], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[94:95], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[0:1], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 18
-; GFX11-NEXT: s_lshr_b32 s42, s10, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 19
-; GFX11-NEXT: s_lshr_b32 s42, s13, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 20
-; GFX11-NEXT: s_lshr_b32 s42, s13, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 21
-; GFX11-NEXT: s_lshr_b32 s42, s13, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 22
-; GFX11-NEXT: s_lshr_b32 s42, s12, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 23
-; GFX11-NEXT: s_lshr_b32 s42, s12, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 24
-; GFX11-NEXT: s_lshr_b32 s42, s15, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 25
-; GFX11-NEXT: s_lshr_b32 s42, s15, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 26
-; GFX11-NEXT: s_lshr_b32 s42, s15, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 27
-; GFX11-NEXT: s_lshr_b32 s42, s14, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 28
-; GFX11-NEXT: s_lshr_b32 s42, s14, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 29
-; GFX11-NEXT: s_lshr_b32 s42, s41, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 30
-; GFX11-NEXT: s_lshr_b32 s42, s41, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 31
-; GFX11-NEXT: s_lshr_b32 s42, s41, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s40, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s40, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s29, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s29, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s29, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s28, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s28, 8
+; GFX11-NEXT: s_lshr_b64 vcc, s[0:1], 24
+; GFX11-NEXT: v_writelane_b32 v26, s75, 7
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 4
+; GFX11-NEXT: v_writelane_b32 v26, s75, 5
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[26:27], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 2
+; GFX11-NEXT: v_writelane_b32 v26, s75, 3
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[24:25], 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 7
-; GFX11-NEXT: s_lshr_b64 s[42:43], s[4:5], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_3
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: v_writelane_b32 v26, s74, 0
+; GFX11-NEXT: v_writelane_b32 v26, s75, 1
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[22:23], 24
+; GFX11-NEXT: s_branch .LBB57_3
+; GFX11-NEXT: .LBB57_2:
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: s_mov_b32 s103, -1
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr70
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr68
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr100
+; GFX11-NEXT: ; implicit-def: $sgpr102
+; GFX11-NEXT: ; implicit-def: $sgpr101
+; GFX11-NEXT: ; implicit-def: $sgpr98
+; GFX11-NEXT: ; implicit-def: $sgpr99
+; GFX11-NEXT: ; implicit-def: $sgpr87
+; GFX11-NEXT: ; implicit-def: $sgpr97
+; GFX11-NEXT: ; implicit-def: $sgpr96
+; GFX11-NEXT: ; implicit-def: $sgpr85
+; GFX11-NEXT: ; implicit-def: $sgpr86
+; GFX11-NEXT: ; implicit-def: $sgpr84
+; GFX11-NEXT: ; implicit-def: $sgpr83
+; GFX11-NEXT: ; implicit-def: $sgpr82
+; GFX11-NEXT: ; implicit-def: $sgpr80
+; GFX11-NEXT: ; implicit-def: $sgpr81
+; GFX11-NEXT: ; implicit-def: $sgpr67
+; GFX11-NEXT: ; implicit-def: $sgpr71
+; GFX11-NEXT: ; implicit-def: $sgpr69
+; GFX11-NEXT: ; implicit-def: $sgpr65
+; GFX11-NEXT: ; implicit-def: $sgpr66
+; GFX11-NEXT: ; implicit-def: $sgpr64
+; GFX11-NEXT: ; implicit-def: $sgpr55
+; GFX11-NEXT: ; implicit-def: $sgpr54
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr52
+; GFX11-NEXT: ; implicit-def: $sgpr53
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr50
+; GFX11-NEXT: ; implicit-def: $sgpr51
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr49
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr104
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: v_writelane_b32 v26, s42, 0
+; GFX11-NEXT: v_writelane_b32 v26, s43, 1
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: v_writelane_b32 v26, s72, 2
+; GFX11-NEXT: v_writelane_b32 v26, s73, 3
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 4
+; GFX11-NEXT: v_writelane_b32 v26, s73, 5
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 6
+; GFX11-NEXT: v_writelane_b32 v26, s73, 7
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 8
+; GFX11-NEXT: v_writelane_b32 v26, s73, 9
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 10
+; GFX11-NEXT: v_writelane_b32 v26, s73, 11
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 12
+; GFX11-NEXT: v_writelane_b32 v26, s73, 13
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 14
+; GFX11-NEXT: v_writelane_b32 v26, s73, 15
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: .LBB57_3: ; %Flow
+; GFX11-NEXT: s_and_b32 s43, s103, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_mov_b32 s103, s31
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_mov_b32 s31, s42
+; GFX11-NEXT: s_mov_b32 s42, s61
+; GFX11-NEXT: s_mov_b32 s43, s48
+; GFX11-NEXT: v_readlane_b32 s48, v26, 18
+; GFX11-NEXT: v_readlane_b32 s61, v26, 19
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -80869,388 +81601,376 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s7, s7, 0
; GFX11-NEXT: s_add_u32 s4, s4, 3
; GFX11-NEXT: s_addc_u32 s5, s5, 0
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s27, 24
-; GFX11-NEXT: s_lshr_b32 s42, s5, 24
-; GFX11-NEXT: s_lshr_b32 s34, s27, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s5, 16
-; GFX11-NEXT: s_lshr_b32 s35, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s26, 16
-; GFX11-NEXT: s_lshr_b32 s36, s26, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s5, 8
-; GFX11-NEXT: s_lshr_b32 s39, s25, 24
-; GFX11-NEXT: s_lshr_b32 s48, s25, 16
-; GFX11-NEXT: s_lshr_b32 s38, s25, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s4, 16
-; GFX11-NEXT: s_lshr_b32 s50, s24, 16
-; GFX11-NEXT: s_lshr_b32 s49, s24, 8
-; GFX11-NEXT: s_lshr_b32 s52, s23, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s4, 8
-; GFX11-NEXT: s_lshr_b32 s53, s23, 16
-; GFX11-NEXT: s_lshr_b32 s51, s23, 8
-; GFX11-NEXT: s_lshr_b32 s69, s22, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s7, 24
-; GFX11-NEXT: s_lshr_b32 s54, s22, 8
-; GFX11-NEXT: s_lshr_b32 s55, s21, 24
-; GFX11-NEXT: s_lshr_b32 s64, s21, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s7, 16
-; GFX11-NEXT: s_lshr_b32 s65, s21, 8
-; GFX11-NEXT: s_lshr_b32 s82, s20, 16
-; GFX11-NEXT: s_lshr_b32 s68, s20, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s7, 8
-; GFX11-NEXT: s_lshr_b32 s66, s19, 24
-; GFX11-NEXT: s_lshr_b32 s67, s19, 16
-; GFX11-NEXT: s_lshr_b32 s85, s19, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 7
+; GFX11-NEXT: s_lshr_b32 s72, s22, 16
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[8:9], 24
+; GFX11-NEXT: v_writelane_b32 v26, s72, 20
+; GFX11-NEXT: s_lshr_b32 s72, s20, 16
+; GFX11-NEXT: s_lshr_b32 s103, s5, 24
+; GFX11-NEXT: s_lshr_b32 s104, s5, 16
+; GFX11-NEXT: s_lshr_b32 s30, s5, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 21
+; GFX11-NEXT: s_lshr_b32 s72, s20, 8
+; GFX11-NEXT: s_lshr_b32 s31, s4, 16
+; GFX11-NEXT: s_lshr_b32 s34, s4, 8
+; GFX11-NEXT: s_lshr_b32 s35, s7, 24
+; GFX11-NEXT: v_writelane_b32 v26, s72, 22
+; GFX11-NEXT: s_lshr_b32 s72, s19, 8
+; GFX11-NEXT: s_lshr_b32 s37, s7, 16
+; GFX11-NEXT: s_lshr_b32 s36, s7, 8
; GFX11-NEXT: s_lshr_b32 s42, s6, 16
-; GFX11-NEXT: s_lshr_b32 s86, s18, 16
-; GFX11-NEXT: s_lshr_b32 s87, s18, 8
-; GFX11-NEXT: s_lshr_b32 s70, s17, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 8
-; GFX11-NEXT: s_lshr_b32 s42, s6, 8
-; GFX11-NEXT: s_lshr_b32 s71, s17, 16
-; GFX11-NEXT: s_lshr_b32 s96, s17, 8
-; GFX11-NEXT: s_lshr_b32 s97, s16, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 9
-; GFX11-NEXT: s_lshr_b32 s42, s9, 24
-; GFX11-NEXT: s_lshr_b32 s98, s16, 8
-; GFX11-NEXT: s_lshr_b32 s80, s3, 24
-; GFX11-NEXT: s_lshr_b32 s81, s3, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 10
-; GFX11-NEXT: s_lshr_b32 s42, s9, 16
-; GFX11-NEXT: s_lshr_b32 s99, s3, 8
-; GFX11-NEXT: s_lshr_b32 s100, s2, 16
-; GFX11-NEXT: s_lshr_b32 s101, s2, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 11
-; GFX11-NEXT: s_lshr_b32 s42, s9, 8
-; GFX11-NEXT: s_lshr_b32 s83, s1, 24
-; GFX11-NEXT: s_lshr_b32 s84, s1, 16
-; GFX11-NEXT: s_lshr_b32 s102, s1, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 12
-; GFX11-NEXT: s_lshr_b32 s42, s8, 16
-; GFX11-NEXT: s_lshr_b32 s103, s0, 16
-; GFX11-NEXT: s_lshr_b32 s104, s0, 8
-; GFX11-NEXT: s_lshr_b64 s[44:45], s[6:7], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 13
-; GFX11-NEXT: s_lshr_b32 s42, s8, 8
-; GFX11-NEXT: s_lshr_b64 s[46:47], s[8:9], 24
-; GFX11-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
-; GFX11-NEXT: s_lshr_b64 s[58:59], s[12:13], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 14
-; GFX11-NEXT: s_lshr_b32 s42, s11, 24
-; GFX11-NEXT: s_lshr_b64 s[60:61], s[14:15], 24
-; GFX11-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
-; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 15
-; GFX11-NEXT: s_lshr_b32 s42, s11, 16
-; GFX11-NEXT: s_lshr_b64 s[76:77], s[26:27], 24
-; GFX11-NEXT: s_lshr_b64 s[72:73], s[24:25], 24
-; GFX11-NEXT: s_lshr_b64 s[78:79], s[22:23], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 16
-; GFX11-NEXT: s_lshr_b32 s42, s11, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 23
+; GFX11-NEXT: s_lshr_b32 s72, s18, 16
+; GFX11-NEXT: s_lshr_b32 s38, s6, 8
+; GFX11-NEXT: s_lshr_b32 s43, s9, 24
+; GFX11-NEXT: s_lshr_b32 s39, s9, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 24
+; GFX11-NEXT: s_lshr_b32 s72, s18, 8
+; GFX11-NEXT: s_lshr_b32 s45, s9, 8
+; GFX11-NEXT: s_lshr_b32 s48, s8, 16
+; GFX11-NEXT: s_lshr_b32 s44, s8, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 25
+; GFX11-NEXT: s_lshr_b32 s72, s17, 24
+; GFX11-NEXT: s_lshr_b32 s47, s11, 24
+; GFX11-NEXT: s_lshr_b32 s49, s11, 16
+; GFX11-NEXT: s_lshr_b32 s46, s11, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 26
+; GFX11-NEXT: s_lshr_b32 s72, s17, 16
+; GFX11-NEXT: s_lshr_b32 s51, s10, 16
+; GFX11-NEXT: s_lshr_b32 s50, s10, 8
+; GFX11-NEXT: s_lshr_b32 s57, s13, 24
+; GFX11-NEXT: v_writelane_b32 v26, s72, 27
+; GFX11-NEXT: s_lshr_b32 s72, s17, 8
+; GFX11-NEXT: s_lshr_b32 s56, s13, 16
+; GFX11-NEXT: s_lshr_b32 s59, s13, 8
+; GFX11-NEXT: s_lshr_b32 s61, s12, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 28
+; GFX11-NEXT: s_lshr_b32 s72, s16, 16
+; GFX11-NEXT: s_lshr_b32 s58, s12, 8
+; GFX11-NEXT: s_lshr_b32 s60, s15, 24
+; GFX11-NEXT: s_lshr_b32 s53, s15, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 29
+; GFX11-NEXT: s_lshr_b32 s72, s16, 8
+; GFX11-NEXT: s_lshr_b32 s52, s15, 8
+; GFX11-NEXT: s_lshr_b32 s62, s14, 16
+; GFX11-NEXT: s_lshr_b32 s63, s14, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 30
+; GFX11-NEXT: s_lshr_b32 s72, s3, 24
+; GFX11-NEXT: s_lshr_b32 s54, s41, 24
+; GFX11-NEXT: s_lshr_b32 s55, s41, 16
+; GFX11-NEXT: s_lshr_b32 s64, s41, 8
+; GFX11-NEXT: v_writelane_b32 v26, s72, 31
+; GFX11-NEXT: s_lshr_b32 s72, s3, 16
+; GFX11-NEXT: s_lshr_b32 s66, s40, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 0
+; GFX11-NEXT: s_lshr_b32 s72, s3, 8
+; GFX11-NEXT: s_lshr_b32 s65, s40, 8
+; GFX11-NEXT: s_lshr_b32 s69, s29, 24
+; GFX11-NEXT: s_lshr_b32 s71, s29, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 1
+; GFX11-NEXT: s_lshr_b32 s72, s2, 16
+; GFX11-NEXT: s_lshr_b32 s67, s29, 8
+; GFX11-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-NEXT: s_lshr_b32 s80, s28, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 2
+; GFX11-NEXT: s_lshr_b32 s72, s2, 8
+; GFX11-NEXT: s_lshr_b32 s82, s27, 24
+; GFX11-NEXT: s_lshr_b32 s83, s27, 16
+; GFX11-NEXT: s_lshr_b32 s84, s27, 8
+; GFX11-NEXT: v_writelane_b32 v25, s72, 3
+; GFX11-NEXT: s_lshr_b32 s72, s1, 24
+; GFX11-NEXT: s_lshr_b32 s86, s26, 16
+; GFX11-NEXT: s_lshr_b32 s85, s26, 8
+; GFX11-NEXT: s_lshr_b32 s96, s25, 24
+; GFX11-NEXT: v_writelane_b32 v25, s72, 4
+; GFX11-NEXT: s_lshr_b32 s72, s1, 16
+; GFX11-NEXT: s_lshr_b32 s97, s25, 16
+; GFX11-NEXT: s_lshr_b32 s87, s25, 8
+; GFX11-NEXT: s_lshr_b32 s99, s24, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 5
+; GFX11-NEXT: s_lshr_b32 s72, s1, 8
+; GFX11-NEXT: s_lshr_b32 s98, s24, 8
+; GFX11-NEXT: s_lshr_b32 s101, s23, 24
+; GFX11-NEXT: s_lshr_b32 s102, s23, 16
+; GFX11-NEXT: v_writelane_b32 v25, s72, 6
+; GFX11-NEXT: s_lshr_b32 s72, s0, 16
+; GFX11-NEXT: s_lshr_b32 s100, s23, 8
+; GFX11-NEXT: s_lshr_b32 s76, s22, 8
+; GFX11-NEXT: s_lshr_b32 s78, s21, 24
+; GFX11-NEXT: v_writelane_b32 v25, s72, 7
+; GFX11-NEXT: s_lshr_b32 s72, s0, 8
+; GFX11-NEXT: s_lshr_b32 s77, s21, 16
+; GFX11-NEXT: s_lshr_b32 s68, s21, 8
+; GFX11-NEXT: s_lshr_b32 s79, s19, 24
+; GFX11-NEXT: v_writelane_b32 v25, s72, 8
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[4:5], 24
+; GFX11-NEXT: s_lshr_b32 s70, s19, 16
+; GFX11-NEXT: v_writelane_b32 v26, s72, 16
; GFX11-NEXT: s_lshr_b64 s[88:89], s[20:21], 24
; GFX11-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
; GFX11-NEXT: s_lshr_b64 s[92:93], s[16:17], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 17
-; GFX11-NEXT: s_lshr_b32 s42, s10, 16
; GFX11-NEXT: s_lshr_b64 s[94:95], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[0:1], 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 18
-; GFX11-NEXT: s_lshr_b32 s42, s10, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 19
-; GFX11-NEXT: s_lshr_b32 s42, s13, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 20
-; GFX11-NEXT: s_lshr_b32 s42, s13, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 21
-; GFX11-NEXT: s_lshr_b32 s42, s13, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 22
-; GFX11-NEXT: s_lshr_b32 s42, s12, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 23
-; GFX11-NEXT: s_lshr_b32 s42, s12, 8
-; GFX11-NEXT: v_writelane_b32 v26, s42, 24
-; GFX11-NEXT: s_lshr_b32 s42, s15, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 25
-; GFX11-NEXT: s_lshr_b32 s42, s15, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 26
-; GFX11-NEXT: s_lshr_b32 s42, s15, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 27
-; GFX11-NEXT: s_lshr_b32 s42, s14, 16
-; GFX11-NEXT: v_writelane_b32 v26, s42, 28
-; GFX11-NEXT: s_lshr_b32 s42, s14, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 29
-; GFX11-NEXT: s_lshr_b32 s42, s41, 24
-; GFX11-NEXT: v_writelane_b32 v26, s42, 30
-; GFX11-NEXT: s_lshr_b32 s42, s41, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v26, s42, 31
-; GFX11-NEXT: s_lshr_b32 s42, s41, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 0
-; GFX11-NEXT: s_lshr_b32 s42, s40, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 1
-; GFX11-NEXT: s_lshr_b32 s42, s40, 8
-; GFX11-NEXT: v_writelane_b32 v25, s42, 2
-; GFX11-NEXT: s_lshr_b32 s42, s29, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 3
-; GFX11-NEXT: s_lshr_b32 s42, s29, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 4
-; GFX11-NEXT: s_lshr_b32 s42, s29, 8
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 5
-; GFX11-NEXT: s_lshr_b32 s42, s28, 16
-; GFX11-NEXT: v_writelane_b32 v25, s42, 6
-; GFX11-NEXT: s_lshr_b32 s42, s28, 8
+; GFX11-NEXT: v_writelane_b32 v26, s73, 17
+; GFX11-NEXT: s_lshr_b64 s[72:73], s[6:7], 24
+; GFX11-NEXT: s_lshr_b64 vcc, s[0:1], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 14
+; GFX11-NEXT: v_writelane_b32 v26, s75, 15
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[10:11], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 12
+; GFX11-NEXT: v_writelane_b32 v26, s75, 13
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[12:13], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 10
+; GFX11-NEXT: v_writelane_b32 v26, s75, 11
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[14:15], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 8
+; GFX11-NEXT: v_writelane_b32 v26, s75, 9
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 6
+; GFX11-NEXT: v_writelane_b32 v26, s75, 7
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[28:29], 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_writelane_b32 v26, s74, 4
+; GFX11-NEXT: v_writelane_b32 v26, s75, 5
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[26:27], 24
+; GFX11-NEXT: v_writelane_b32 v26, s74, 2
+; GFX11-NEXT: v_writelane_b32 v26, s75, 3
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[24:25], 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_writelane_b32 v25, s42, 7
-; GFX11-NEXT: s_lshr_b64 s[42:43], s[4:5], 24
-; GFX11-NEXT: .LBB57_3: ; %end
+; GFX11-NEXT: v_writelane_b32 v26, s74, 0
+; GFX11-NEXT: v_writelane_b32 v26, s75, 1
+; GFX11-NEXT: s_lshr_b64 s[74:75], s[22:23], 24
+; GFX11-NEXT: .LBB57_5: ; %end
; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, s103, s30, v1
-; GFX11-NEXT: v_readlane_b32 s30, v24, 7
-; GFX11-NEXT: v_readlane_b32 s31, v24, 8
-; GFX11-NEXT: v_readlane_b32 s103, v24, 5
+; GFX11-NEXT: v_readlane_b32 s73, v25, 7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v2, s73, vcc_lo, v1
+; GFX11-NEXT: v_readlane_b32 s73, v25, 8
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_perm_b32 v18, s84, s83, v1
-; GFX11-NEXT: v_perm_b32 v3, s0, s104, v1
-; GFX11-NEXT: v_perm_b32 v4, s1, s102, v1
-; GFX11-NEXT: v_perm_b32 v5, s100, s94, v1
-; GFX11-NEXT: v_perm_b32 v19, s81, s80, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-NEXT: v_perm_b32 v6, s2, s101, v1
-; GFX11-NEXT: v_perm_b32 v7, s3, s99, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v3, s0, s73, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 6
+; GFX11-NEXT: v_perm_b32 v21, s70, s79, v1
+; GFX11-NEXT: v_perm_b32 v22, s77, s78, v1
+; GFX11-NEXT: v_perm_b32 v17, s21, s68, v1
; GFX11-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11-NEXT: v_perm_b32 v8, s97, s92, v1
+; GFX11-NEXT: v_perm_b32 v4, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 2
+; GFX11-NEXT: v_readlane_b32 s1, v25, 5
+; GFX11-NEXT: v_readlane_b32 s70, v23, 20
+; GFX11-NEXT: v_readlane_b32 s68, v23, 18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v5, s0, s94, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 3
+; GFX11-NEXT: v_perm_b32 v6, s2, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v7, s3, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 29
+; GFX11-NEXT: v_perm_b32 v8, s0, s92, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 30
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v9, s16, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 28
+; GFX11-NEXT: v_perm_b32 v10, s17, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 24
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v11, s0, s90, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 25
+; GFX11-NEXT: v_perm_b32 v12, s18, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v13, s19, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 21
+; GFX11-NEXT: v_perm_b32 v14, s0, s88, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, s0, s74, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 22
+; GFX11-NEXT: v_perm_b32 v16, s20, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v25, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v18, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 31
+; GFX11-NEXT: v_readlane_b32 s1, v25, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v19, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 26
+; GFX11-NEXT: v_readlane_b32 s1, v26, 27
; GFX11-NEXT: v_or_b32_e32 v3, v4, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v19
-; GFX11-NEXT: v_perm_b32 v11, s86, s90, v1
-; GFX11-NEXT: v_perm_b32 v20, s71, s70, v1
-; GFX11-NEXT: v_perm_b32 v21, s67, s66, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v20, s1, s0, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 0
; GFX11-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v5, v7, v5
-; GFX11-NEXT: v_perm_b32 v9, s16, s98, v1
-; GFX11-NEXT: v_perm_b32 v10, s17, s96, v1
-; GFX11-NEXT: v_perm_b32 v12, s18, s87, v1
-; GFX11-NEXT: v_perm_b32 v13, s19, s85, v1
-; GFX11-NEXT: v_perm_b32 v22, s64, s55, v1
+; GFX11-NEXT: v_perm_b32 v6, s102, s101, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
+; GFX11-NEXT: v_readlane_b32 s1, v26, 1
+; GFX11-NEXT: v_perm_b32 v14, s7, s36, v1
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v21
-; GFX11-NEXT: v_perm_b32 v14, s82, s88, v1
-; GFX11-NEXT: v_perm_b32 v17, s21, s65, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v22
; GFX11-NEXT: v_or_b32_e32 v2, v9, v2
; GFX11-NEXT: v_or_b32_e32 v3, v10, v3
; GFX11-NEXT: v_or_b32_e32 v4, v12, v4
; GFX11-NEXT: v_or_b32_e32 v5, v13, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v22
-; GFX11-NEXT: v_perm_b32 v15, s69, s78, v1
-; GFX11-NEXT: v_perm_b32 v16, s20, s68, v1
-; GFX11-NEXT: v_perm_b32 v6, s53, s52, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:16
-; GFX11-NEXT: v_or_b32_e32 v3, v17, v8
-; GFX11-NEXT: v_perm_b32 v8, s48, s39, v1
-; GFX11-NEXT: v_perm_b32 v9, s22, s54, v1
+; GFX11-NEXT: v_perm_b32 v9, s22, s76, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v15
-; GFX11-NEXT: v_or_b32_e32 v2, v16, v7
-; GFX11-NEXT: v_perm_b32 v5, s50, s72, v1
-; GFX11-NEXT: v_perm_b32 v7, s23, s51, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_perm_b32 v11, s25, s38, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-NEXT: v_readlane_b32 s0, v25, 6
+; GFX11-NEXT: v_perm_b32 v11, s25, s87, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:16
+; GFX11-NEXT: v_or_b32_e32 v2, v16, v7
+; GFX11-NEXT: v_or_b32_e32 v3, v17, v8
+; GFX11-NEXT: v_perm_b32 v5, s99, s0, v1
+; GFX11-NEXT: v_perm_b32 v7, s23, s100, v1
+; GFX11-NEXT: v_perm_b32 v8, s97, s96, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 2
+; GFX11-NEXT: v_readlane_b32 s1, v26, 3
; GFX11-NEXT: v_or_b32_e32 v4, v9, v10
+; GFX11-NEXT: v_perm_b32 v9, s24, s98, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-NEXT: v_perm_b32 v12, s86, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v7, v6
-; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
-; GFX11-NEXT: v_perm_b32 v11, s0, s74, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 3
-; GFX11-NEXT: v_readlane_b32 s1, v25, 4
-; GFX11-NEXT: v_perm_b32 v9, s24, s49, v1
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:32
-; GFX11-NEXT: v_perm_b32 v12, s37, s76, v1
-; GFX11-NEXT: v_perm_b32 v8, s26, s36, v1
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 7
+; GFX11-NEXT: v_readlane_b32 s0, v26, 4
+; GFX11-NEXT: v_readlane_b32 s1, v26, 5
; GFX11-NEXT: v_or_b32_e32 v6, v9, v10
-; GFX11-NEXT: v_perm_b32 v10, s34, vcc_hi, v1
+; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
+; GFX11-NEXT: v_perm_b32 v8, s26, s85, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; GFX11-NEXT: v_perm_b32 v4, s28, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 5
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v10, s83, s82, v1
+; GFX11-NEXT: v_perm_b32 v11, s81, s0, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:32
+; GFX11-NEXT: v_perm_b32 v2, s71, s69, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 6
; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX11-NEXT: v_perm_b32 v9, s27, s35, v1
+; GFX11-NEXT: v_perm_b32 v9, s27, s84, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v4, s28, s80, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s29, s67, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_perm_b32 v10, s29, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 1
-; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_readlane_b32 s1, v26, 31
+; GFX11-NEXT: v_perm_b32 v12, s66, s0, v1
+; GFX11-NEXT: v_readlane_b32 s1, v26, 7
; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s62, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 2
-; GFX11-NEXT: v_readlane_b32 s104, v24, 6
+; GFX11-NEXT: v_perm_b32 v4, s40, s65, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
+; GFX11-NEXT: v_perm_b32 v10, s55, s54, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 8
+; GFX11-NEXT: v_readlane_b32 s1, v26, 9
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:48
-; GFX11-NEXT: v_readlane_b32 s102, v24, 4
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_perm_b32 v5, s41, s64, v1
+; GFX11-NEXT: v_perm_b32 v11, s62, s0, v1
+; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
+; GFX11-NEXT: v_readlane_b32 s0, v26, 10
+; GFX11-NEXT: v_readlane_b32 s1, v26, 11
+; GFX11-NEXT: v_perm_b32 v8, s14, s63, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s15, s52, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-NEXT: v_perm_b32 v12, s61, s0, v1
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-NEXT: v_readlane_b32 s0, v26, 12
+; GFX11-NEXT: v_readlane_b32 s1, v26, 13
+; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
+; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
+; GFX11-NEXT: v_perm_b32 v8, s12, s58, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-NEXT: v_perm_b32 v10, s56, s57, v1
+; GFX11-NEXT: v_perm_b32 v11, s51, s0, v1
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:64
+; GFX11-NEXT: v_perm_b32 v2, s49, s47, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 14
+; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX11-NEXT: v_perm_b32 v9, s13, s59, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-NEXT: v_perm_b32 v4, s10, s50, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; GFX11-NEXT: v_perm_b32 v10, s11, s46, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-NEXT: v_perm_b32 v12, s48, s0, v1
+; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
+; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
+; GFX11-NEXT: v_perm_b32 v4, s8, s44, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
-; GFX11-NEXT: v_perm_b32 v4, s40, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 30
+; GFX11-NEXT: v_perm_b32 v10, s39, s43, v1
+; GFX11-NEXT: v_perm_b32 v11, s42, s72, v1
+; GFX11-NEXT: v_readlane_b32 s1, v26, 15
+; GFX11-NEXT: v_perm_b32 v12, s6, s38, v1
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_perm_b32 v5, s9, s45, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-NEXT: v_readlane_b32 s0, v26, 16
+; GFX11-NEXT: v_readlane_b32 s1, v26, 17
+; GFX11-NEXT: v_readlane_b32 s102, v24, 4
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
+; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s101, v24, 3
; GFX11-NEXT: v_readlane_b32 s100, v24, 2
; GFX11-NEXT: v_readlane_b32 s99, v24, 1
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v25, 0
-; GFX11-NEXT: v_readlane_b32 s1, v26, 26
; GFX11-NEXT: v_readlane_b32 s98, v24, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
+; GFX11-NEXT: v_perm_b32 v12, s31, s0, v1
; GFX11-NEXT: v_readlane_b32 s97, v23, 31
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
-; GFX11-NEXT: v_perm_b32 v5, s41, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 28
; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
+; GFX11-NEXT: v_perm_b32 v13, s4, s34, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-NEXT: v_readlane_b32 s87, v23, 29
; GFX11-NEXT: v_readlane_b32 s86, v23, 28
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-NEXT: v_perm_b32 v11, s0, s60, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 25
; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_readlane_b32 s84, v23, 26
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:64
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
-; GFX11-NEXT: v_perm_b32 v6, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 29
-; GFX11-NEXT: v_readlane_b32 s1, v26, 21
+; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
+; GFX11-NEXT: v_perm_b32 v13, s104, s103, v1
+; GFX11-NEXT: v_perm_b32 v1, s5, s30, v1
+; GFX11-NEXT: v_readlane_b32 s30, v24, 7
+; GFX11-NEXT: v_readlane_b32 s31, v24, 8
+; GFX11-NEXT: v_readlane_b32 s104, v24, 6
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT: v_readlane_b32 s103, v24, 5
; GFX11-NEXT: v_readlane_b32 s83, v23, 25
; GFX11-NEXT: v_readlane_b32 s82, v23, 24
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-NEXT: v_perm_b32 v8, s14, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 27
; GFX11-NEXT: v_readlane_b32 s81, v23, 23
+; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
+; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
; GFX11-NEXT: v_readlane_b32 s80, v23, 22
; GFX11-NEXT: v_readlane_b32 s71, v23, 21
-; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
-; GFX11-NEXT: v_perm_b32 v10, s15, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 23
-; GFX11-NEXT: v_readlane_b32 s70, v23, 20
; GFX11-NEXT: v_readlane_b32 s69, v23, 19
-; GFX11-NEXT: v_readlane_b32 s68, v23, 18
-; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s58, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 24
; GFX11-NEXT: v_readlane_b32 s67, v23, 17
; GFX11-NEXT: v_readlane_b32 s66, v23, 16
; GFX11-NEXT: v_readlane_b32 s65, v23, 15
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-NEXT: v_perm_b32 v8, s12, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 20
; GFX11-NEXT: v_readlane_b32 s64, v23, 14
; GFX11-NEXT: v_readlane_b32 s55, v23, 13
; GFX11-NEXT: v_readlane_b32 s54, v23, 12
-; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 22
-; GFX11-NEXT: v_readlane_b32 s1, v26, 16
; GFX11-NEXT: v_readlane_b32 s53, v23, 11
; GFX11-NEXT: v_readlane_b32 s52, v23, 10
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v10
-; GFX11-NEXT: v_perm_b32 v9, s13, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 18
; GFX11-NEXT: v_readlane_b32 s51, v23, 9
; GFX11-NEXT: v_readlane_b32 s50, v23, 8
; GFX11-NEXT: v_readlane_b32 s49, v23, 7
-; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
-; GFX11-NEXT: v_perm_b32 v11, s0, s56, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 15
; GFX11-NEXT: v_readlane_b32 s48, v23, 6
; GFX11-NEXT: v_readlane_b32 s39, v23, 5
; GFX11-NEXT: v_readlane_b32 s38, v23, 4
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 19
-; GFX11-NEXT: v_readlane_b32 s1, v26, 11
; GFX11-NEXT: v_readlane_b32 s37, v23, 3
; GFX11-NEXT: v_readlane_b32 s36, v23, 2
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_perm_b32 v4, s10, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 17
; GFX11-NEXT: v_readlane_b32 s35, v23, 1
; GFX11-NEXT: v_readlane_b32 s34, v23, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s11, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 13
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
-; GFX11-NEXT: v_perm_b32 v12, s0, s46, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
-; GFX11-NEXT: v_perm_b32 v4, s8, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 12
-; GFX11-NEXT: v_readlane_b32 s1, v26, 6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-NEXT: v_perm_b32 v5, s9, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
-; GFX11-NEXT: v_perm_b32 v11, s0, s44, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-NEXT: v_perm_b32 v10, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 9
-; GFX11-NEXT: v_readlane_b32 s1, v26, 1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-NEXT: v_perm_b32 v12, s6, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
-; GFX11-NEXT: v_perm_b32 v14, s7, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
-; GFX11-NEXT: v_perm_b32 v12, s0, s42, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT: v_perm_b32 v13, s4, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
-; GFX11-NEXT: v_perm_b32 v13, s1, s0, v1
-; GFX11-NEXT: v_readlane_b32 s0, v26, 2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-NEXT: v_perm_b32 v1, s5, s0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
-; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
-; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
; GFX11-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-NEXT: s_clause 0x3 ; 16-byte Folded Reload
; GFX11-NEXT: scratch_load_b32 v23, off, s32
@@ -81260,144 +81980,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_mov_b32 exec_lo, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_4:
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr104
-; GFX11-NEXT: ; implicit-def: $sgpr103
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $sgpr102
-; GFX11-NEXT: ; implicit-def: $sgpr84
-; GFX11-NEXT: ; implicit-def: $sgpr83
-; GFX11-NEXT: ; implicit-def: $sgpr101
-; GFX11-NEXT: ; implicit-def: $sgpr100
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr99
-; GFX11-NEXT: ; implicit-def: $sgpr81
-; GFX11-NEXT: ; implicit-def: $sgpr80
-; GFX11-NEXT: ; implicit-def: $sgpr98
-; GFX11-NEXT: ; implicit-def: $sgpr97
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr96
-; GFX11-NEXT: ; implicit-def: $sgpr71
-; GFX11-NEXT: ; implicit-def: $sgpr70
-; GFX11-NEXT: ; implicit-def: $sgpr87
-; GFX11-NEXT: ; implicit-def: $sgpr86
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr85
-; GFX11-NEXT: ; implicit-def: $sgpr67
-; GFX11-NEXT: ; implicit-def: $sgpr66
-; GFX11-NEXT: ; implicit-def: $sgpr68
-; GFX11-NEXT: ; implicit-def: $sgpr82
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr65
-; GFX11-NEXT: ; implicit-def: $sgpr64
-; GFX11-NEXT: ; implicit-def: $sgpr55
-; GFX11-NEXT: ; implicit-def: $sgpr54
-; GFX11-NEXT: ; implicit-def: $sgpr69
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr51
-; GFX11-NEXT: ; implicit-def: $sgpr53
-; GFX11-NEXT: ; implicit-def: $sgpr52
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr50
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: s_branch .LBB57_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -87913,8 +88495,10 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_cbranch_vccnz .LBB59_5
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
@@ -88911,8 +89495,10 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; VI-NEXT: s_cbranch_vccnz .LBB59_5
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
@@ -89835,8 +90421,10 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB59_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
@@ -90331,7 +90919,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -90477,9 +91065,17 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_3
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB59_3
+; GFX11-TRUE16-NEXT: .LBB59_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB59_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
@@ -90770,15 +91366,12 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-TRUE16-NEXT: .LBB59_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB59_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB59_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v16i64_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -90905,7 +91498,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -91051,9 +91644,17 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_3
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB59_3
+; GFX11-FAKE16-NEXT: .LBB59_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB59_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
@@ -91344,15 +91945,12 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-FAKE16-NEXT: .LBB59_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB59_5: ; %end
; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB59_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -92223,381 +92821,100 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; SI-NEXT: v_writelane_b32 v33, s30, 34
; SI-NEXT: v_writelane_b32 v33, s31, 35
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_readfirstlane_b32 s70, v17
-; SI-NEXT: v_readfirstlane_b32 s71, v16
-; SI-NEXT: v_readfirstlane_b32 s80, v15
-; SI-NEXT: v_readfirstlane_b32 s81, v14
-; SI-NEXT: v_readfirstlane_b32 s82, v13
-; SI-NEXT: v_readfirstlane_b32 s83, v12
-; SI-NEXT: v_readfirstlane_b32 s84, v11
-; SI-NEXT: v_readfirstlane_b32 s85, v10
-; SI-NEXT: v_readfirstlane_b32 s86, v9
-; SI-NEXT: v_readfirstlane_b32 s87, v8
-; SI-NEXT: v_readfirstlane_b32 s96, v7
-; SI-NEXT: v_readfirstlane_b32 s97, v6
-; SI-NEXT: v_readfirstlane_b32 s98, v5
-; SI-NEXT: v_readfirstlane_b32 s99, v4
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_readfirstlane_b32 s7, v2
-; SI-NEXT: v_readfirstlane_b32 s8, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v17
+; SI-NEXT: v_readfirstlane_b32 s69, v16
+; SI-NEXT: v_readfirstlane_b32 s70, v15
+; SI-NEXT: v_readfirstlane_b32 s71, v14
+; SI-NEXT: v_readfirstlane_b32 s80, v13
+; SI-NEXT: v_readfirstlane_b32 s81, v12
+; SI-NEXT: v_readfirstlane_b32 s82, v11
+; SI-NEXT: v_readfirstlane_b32 s83, v10
+; SI-NEXT: v_readfirstlane_b32 s84, v9
+; SI-NEXT: v_readfirstlane_b32 s85, v8
+; SI-NEXT: v_readfirstlane_b32 s86, v7
+; SI-NEXT: v_readfirstlane_b32 s87, v6
+; SI-NEXT: v_readfirstlane_b32 s96, v5
+; SI-NEXT: v_readfirstlane_b32 s97, v4
+; SI-NEXT: v_readfirstlane_b32 s98, v3
+; SI-NEXT: v_readfirstlane_b32 s99, v2
+; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s9, v0
+; SI-NEXT: v_readfirstlane_b32 s7, v0
; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB61_4
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s70, 0xffff0000
+; SI-NEXT: s_and_b32 s4, s68, 0xffff0000
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v34, s4, 0
-; SI-NEXT: s_lshl_b32 s4, s70, 16
+; SI-NEXT: s_lshl_b32 s4, s68, 16
; SI-NEXT: v_writelane_b32 v34, s4, 1
-; SI-NEXT: s_and_b32 s4, s71, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s4, 2
-; SI-NEXT: s_lshl_b32 s4, s71, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 3
-; SI-NEXT: s_and_b32 s11, s80, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s80, 16
-; SI-NEXT: s_and_b32 s13, s81, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s81, 16
-; SI-NEXT: s_and_b32 s15, s82, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s82, 16
-; SI-NEXT: s_and_b32 s41, s83, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s83, 16
-; SI-NEXT: s_and_b32 s43, s84, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s84, 16
-; SI-NEXT: s_and_b32 s45, s85, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s85, 16
-; SI-NEXT: s_and_b32 s47, s86, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s86, 16
-; SI-NEXT: s_and_b32 s57, s87, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s87, 16
-; SI-NEXT: s_and_b32 s59, s96, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s96, 16
-; SI-NEXT: s_and_b32 s61, s97, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s97, 16
-; SI-NEXT: s_and_b32 s63, s98, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s98, 16
-; SI-NEXT: s_and_b32 s73, s99, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s99, 16
-; SI-NEXT: s_and_b32 s75, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s6, 16
-; SI-NEXT: s_and_b32 s77, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s7, 16
-; SI-NEXT: s_and_b32 s79, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s8, 16
-; SI-NEXT: s_and_b32 s89, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s9, 16
+; SI-NEXT: s_and_b32 s9, s69, 0xffff0000
+; SI-NEXT: s_lshl_b32 s8, s69, 16
+; SI-NEXT: s_and_b32 s11, s70, 0xffff0000
+; SI-NEXT: s_lshl_b32 s10, s70, 16
+; SI-NEXT: s_and_b32 s13, s71, 0xffff0000
+; SI-NEXT: s_lshl_b32 s12, s71, 16
+; SI-NEXT: s_and_b32 s15, s80, 0xffff0000
+; SI-NEXT: s_lshl_b32 s14, s80, 16
+; SI-NEXT: s_and_b32 s41, s81, 0xffff0000
+; SI-NEXT: s_lshl_b32 s40, s81, 16
+; SI-NEXT: s_and_b32 s43, s82, 0xffff0000
+; SI-NEXT: s_lshl_b32 s42, s82, 16
+; SI-NEXT: s_and_b32 s45, s83, 0xffff0000
+; SI-NEXT: s_lshl_b32 s44, s83, 16
+; SI-NEXT: s_and_b32 s47, s84, 0xffff0000
+; SI-NEXT: s_lshl_b32 s46, s84, 16
+; SI-NEXT: s_and_b32 s57, s85, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s85, 16
+; SI-NEXT: s_and_b32 s59, s86, 0xffff0000
+; SI-NEXT: s_lshl_b32 s58, s86, 16
+; SI-NEXT: s_and_b32 s61, s87, 0xffff0000
+; SI-NEXT: s_lshl_b32 s60, s87, 16
+; SI-NEXT: s_and_b32 s63, s96, 0xffff0000
+; SI-NEXT: s_lshl_b32 s62, s96, 16
+; SI-NEXT: s_and_b32 s73, s97, 0xffff0000
+; SI-NEXT: s_lshl_b32 s72, s97, 16
+; SI-NEXT: s_and_b32 s75, s98, 0xffff0000
+; SI-NEXT: s_lshl_b32 s74, s98, 16
+; SI-NEXT: s_and_b32 s77, s99, 0xffff0000
+; SI-NEXT: s_lshl_b32 s76, s99, 16
+; SI-NEXT: s_and_b32 s79, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s78, s6, 16
+; SI-NEXT: s_and_b32 s89, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s7, 16
; SI-NEXT: s_and_b32 s91, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s90, s29, 16
; SI-NEXT: s_and_b32 s93, s28, 0xffff0000
; SI-NEXT: s_lshl_b32 s92, s28, 16
; SI-NEXT: s_and_b32 s95, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s27, 16
-; SI-NEXT: s_and_b32 s31, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s26, 16
-; SI-NEXT: s_and_b32 s35, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s25, 16
-; SI-NEXT: s_and_b32 s37, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s24, 16
-; SI-NEXT: s_and_b32 s39, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s23, 16
-; SI-NEXT: s_and_b32 s49, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s22, 16
-; SI-NEXT: s_and_b32 s51, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s21, 16
-; SI-NEXT: s_and_b32 s53, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s20, 16
-; SI-NEXT: s_and_b32 s55, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s19, 16
-; SI-NEXT: s_and_b32 s65, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s18, 16
-; SI-NEXT: s_and_b32 s67, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s17, 16
-; SI-NEXT: s_and_b32 s69, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB61_3
-; SI-NEXT: .LBB61_2: ; %cmp.true
-; SI-NEXT: s_add_u32 s4, s16, 3
-; SI-NEXT: s_addc_u32 s5, s17, 0
-; SI-NEXT: s_add_u32 s16, s18, 3
-; SI-NEXT: s_addc_u32 s17, s19, 0
-; SI-NEXT: s_add_u32 s18, s20, 3
-; SI-NEXT: s_addc_u32 s19, s21, 0
-; SI-NEXT: s_add_u32 s20, s22, 3
-; SI-NEXT: s_addc_u32 s21, s23, 0
-; SI-NEXT: s_add_u32 s22, s24, 3
-; SI-NEXT: s_addc_u32 s23, s25, 0
-; SI-NEXT: s_add_u32 s24, s26, 3
-; SI-NEXT: s_addc_u32 s25, s27, 0
-; SI-NEXT: s_add_u32 s26, s28, 3
-; SI-NEXT: s_addc_u32 s27, s29, 0
-; SI-NEXT: s_add_u32 s9, s9, 3
-; SI-NEXT: s_addc_u32 s8, s8, 0
-; SI-NEXT: s_add_u32 s7, s7, 3
-; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: s_add_u32 s28, s99, 3
-; SI-NEXT: s_addc_u32 s29, s98, 0
-; SI-NEXT: s_add_u32 s60, s97, 3
-; SI-NEXT: s_addc_u32 s58, s96, 0
-; SI-NEXT: s_add_u32 s56, s87, 3
-; SI-NEXT: s_addc_u32 s46, s86, 0
-; SI-NEXT: s_add_u32 s44, s85, 3
-; SI-NEXT: s_addc_u32 s42, s84, 0
-; SI-NEXT: s_add_u32 s40, s83, 3
-; SI-NEXT: s_addc_u32 s14, s82, 0
-; SI-NEXT: s_add_u32 s12, s81, 3
-; SI-NEXT: s_addc_u32 s10, s80, 0
-; SI-NEXT: s_add_u32 s11, s71, 3
-; SI-NEXT: s_addc_u32 s13, s70, 0
-; SI-NEXT: s_and_b32 s15, s13, 0xffff0000
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v34, s15, 0
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: v_writelane_b32 v34, s13, 1
-; SI-NEXT: s_and_b32 s13, s11, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s13, 2
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: v_writelane_b32 v34, s11, 3
-; SI-NEXT: s_and_b32 s11, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_and_b32 s13, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s15, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_and_b32 s41, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: s_lshl_b32 s60, s60, 16
-; SI-NEXT: s_and_b32 s63, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s62, s29, 16
-; SI-NEXT: s_and_b32 s73, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s72, s28, 16
-; SI-NEXT: s_and_b32 s75, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s6, 16
-; SI-NEXT: s_and_b32 s77, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s76, s7, 16
-; SI-NEXT: s_and_b32 s79, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s8, 16
-; SI-NEXT: s_and_b32 s89, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s9, 16
-; SI-NEXT: s_and_b32 s91, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s27, 16
-; SI-NEXT: s_and_b32 s93, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s26, 16
-; SI-NEXT: s_and_b32 s95, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s25, 16
-; SI-NEXT: s_and_b32 s31, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s24, 16
-; SI-NEXT: s_and_b32 s35, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s23, 16
-; SI-NEXT: s_and_b32 s37, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s22, 16
-; SI-NEXT: s_and_b32 s39, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s21, 16
-; SI-NEXT: s_and_b32 s49, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s20, 16
-; SI-NEXT: s_and_b32 s51, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s19, 16
-; SI-NEXT: s_and_b32 s53, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s18, 16
-; SI-NEXT: s_and_b32 s55, s17, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s17, 16
-; SI-NEXT: s_and_b32 s65, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s16, 16
-; SI-NEXT: s_and_b32 s67, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s5, 16
-; SI-NEXT: s_and_b32 s69, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s4, 16
-; SI-NEXT: .LBB61_3: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s69
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s68
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s67
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s66
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s65
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s64
-; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s55
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s54
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s53
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s52
-; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s51
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s50
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s49
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s48
-; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s39
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s38
-; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s37
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s36
-; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s35
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s34
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s31
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s30
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s95
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s94
-; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s93
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s92
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s91
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s90
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s89
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s88
-; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s79
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s78
-; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s77
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s76
-; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s75
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s74
-; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s73
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s72
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s63
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s62
-; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s61
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s60
-; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s59
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s58
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s57
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s56
-; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s47
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
-; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s45
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s44
-; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s43
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s42
-; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s41
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s40
-; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s15
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s14
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s13
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
-; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_readlane_b32 s4, v34, 2
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s4
-; SI-NEXT: v_readlane_b32 s4, v34, 3
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s4
-; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_readlane_b32 s4, v34, 0
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_readlane_b32 s4, v34, 1
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
-; SI-NEXT: v_readlane_b32 s30, v33, 34
-; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
-; SI-NEXT: v_readlane_b32 s31, v33, 35
-; SI-NEXT: v_readlane_b32 s99, v33, 33
-; SI-NEXT: v_readlane_b32 s98, v33, 32
-; SI-NEXT: v_readlane_b32 s97, v33, 31
-; SI-NEXT: v_readlane_b32 s96, v33, 30
-; SI-NEXT: v_readlane_b32 s87, v33, 29
-; SI-NEXT: v_readlane_b32 s86, v33, 28
-; SI-NEXT: v_readlane_b32 s85, v33, 27
-; SI-NEXT: v_readlane_b32 s84, v33, 26
-; SI-NEXT: v_readlane_b32 s83, v33, 25
-; SI-NEXT: v_readlane_b32 s82, v33, 24
-; SI-NEXT: v_readlane_b32 s81, v33, 23
-; SI-NEXT: v_readlane_b32 s80, v33, 22
-; SI-NEXT: v_readlane_b32 s71, v33, 21
-; SI-NEXT: v_readlane_b32 s70, v33, 20
-; SI-NEXT: v_readlane_b32 s69, v33, 19
-; SI-NEXT: v_readlane_b32 s68, v33, 18
-; SI-NEXT: v_readlane_b32 s67, v33, 17
-; SI-NEXT: v_readlane_b32 s66, v33, 16
-; SI-NEXT: v_readlane_b32 s65, v33, 15
-; SI-NEXT: v_readlane_b32 s64, v33, 14
-; SI-NEXT: v_readlane_b32 s55, v33, 13
-; SI-NEXT: v_readlane_b32 s54, v33, 12
-; SI-NEXT: v_readlane_b32 s53, v33, 11
-; SI-NEXT: v_readlane_b32 s52, v33, 10
-; SI-NEXT: v_readlane_b32 s51, v33, 9
-; SI-NEXT: v_readlane_b32 s50, v33, 8
-; SI-NEXT: v_readlane_b32 s49, v33, 7
-; SI-NEXT: v_readlane_b32 s48, v33, 6
-; SI-NEXT: v_readlane_b32 s39, v33, 5
-; SI-NEXT: v_readlane_b32 s38, v33, 4
-; SI-NEXT: v_readlane_b32 s37, v33, 3
-; SI-NEXT: v_readlane_b32 s36, v33, 2
-; SI-NEXT: v_readlane_b32 s35, v33, 1
-; SI-NEXT: v_readlane_b32 s34, v33, 0
-; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB61_4:
+; SI-NEXT: s_and_b32 vcc_hi, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, s26, 16
+; SI-NEXT: s_and_b32 s31, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s30, s25, 16
+; SI-NEXT: s_and_b32 s35, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s24, 16
+; SI-NEXT: s_and_b32 s37, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s36, s23, 16
+; SI-NEXT: s_and_b32 s39, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s38, s22, 16
+; SI-NEXT: s_and_b32 s49, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s48, s21, 16
+; SI-NEXT: s_and_b32 s51, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s50, s20, 16
+; SI-NEXT: s_and_b32 s53, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s19, 16
+; SI-NEXT: s_and_b32 s55, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s18, 16
+; SI-NEXT: s_and_b32 s65, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s64, s17, 16
+; SI-NEXT: s_and_b32 s67, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s66, s16, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr67
; SI-NEXT: ; implicit-def: $sgpr64
@@ -92618,6 +92935,8 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; SI-NEXT: ; implicit-def: $sgpr35
; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr92
@@ -92656,13 +92975,290 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; SI-NEXT: ; implicit-def: $sgpr13
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: s_branch .LBB61_2
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_u32 s4, s16, 3
+; SI-NEXT: s_addc_u32 s5, s17, 0
+; SI-NEXT: s_add_u32 s16, s18, 3
+; SI-NEXT: s_addc_u32 s17, s19, 0
+; SI-NEXT: s_add_u32 s18, s20, 3
+; SI-NEXT: s_addc_u32 s19, s21, 0
+; SI-NEXT: s_add_u32 s20, s22, 3
+; SI-NEXT: s_addc_u32 s21, s23, 0
+; SI-NEXT: s_add_u32 s22, s24, 3
+; SI-NEXT: s_addc_u32 s23, s25, 0
+; SI-NEXT: s_add_u32 s24, s26, 3
+; SI-NEXT: s_addc_u32 s25, s27, 0
+; SI-NEXT: s_add_u32 s26, s28, 3
+; SI-NEXT: s_addc_u32 s27, s29, 0
+; SI-NEXT: s_add_u32 s7, s7, 3
+; SI-NEXT: s_addc_u32 s6, s6, 0
+; SI-NEXT: s_add_u32 s28, s99, 3
+; SI-NEXT: s_addc_u32 s29, s98, 0
+; SI-NEXT: s_add_u32 s72, s97, 3
+; SI-NEXT: s_addc_u32 s62, s96, 0
+; SI-NEXT: s_add_u32 s60, s87, 3
+; SI-NEXT: s_addc_u32 s58, s86, 0
+; SI-NEXT: s_add_u32 s56, s85, 3
+; SI-NEXT: s_addc_u32 s46, s84, 0
+; SI-NEXT: s_add_u32 s44, s83, 3
+; SI-NEXT: s_addc_u32 s42, s82, 0
+; SI-NEXT: s_add_u32 s40, s81, 3
+; SI-NEXT: s_addc_u32 s14, s80, 0
+; SI-NEXT: s_add_u32 s12, s71, 3
+; SI-NEXT: s_addc_u32 s10, s70, 0
+; SI-NEXT: s_add_u32 s8, s69, 3
+; SI-NEXT: s_addc_u32 s9, s68, 0
+; SI-NEXT: s_and_b32 s11, s9, 0xffff0000
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v34, s11, 0
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: v_writelane_b32 v34, s9, 1
+; SI-NEXT: s_and_b32 s9, s8, 0xffff0000
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_and_b32 s11, s10, 0xffff0000
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_and_b32 s13, s12, 0xffff0000
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_and_b32 s15, s14, 0xffff0000
+; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_and_b32 s41, s40, 0xffff0000
+; SI-NEXT: s_lshl_b32 s40, s40, 16
+; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
+; SI-NEXT: s_lshl_b32 s42, s42, 16
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
+; SI-NEXT: s_lshl_b32 s44, s44, 16
+; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
+; SI-NEXT: s_lshl_b32 s58, s58, 16
+; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
+; SI-NEXT: s_lshl_b32 s60, s60, 16
+; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
+; SI-NEXT: s_lshl_b32 s62, s62, 16
+; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
+; SI-NEXT: s_lshl_b32 s72, s72, 16
+; SI-NEXT: s_and_b32 s75, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s74, s29, 16
+; SI-NEXT: s_and_b32 s77, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 s76, s28, 16
+; SI-NEXT: s_and_b32 s79, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s78, s6, 16
+; SI-NEXT: s_and_b32 s89, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s7, 16
+; SI-NEXT: s_and_b32 s91, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 s90, s27, 16
+; SI-NEXT: s_and_b32 s93, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 s92, s26, 16
+; SI-NEXT: s_and_b32 s95, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s94, s25, 16
+; SI-NEXT: s_and_b32 vcc_hi, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, s24, 16
+; SI-NEXT: s_and_b32 s31, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s30, s23, 16
+; SI-NEXT: s_and_b32 s35, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s22, 16
+; SI-NEXT: s_and_b32 s37, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s36, s21, 16
+; SI-NEXT: s_and_b32 s39, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s38, s20, 16
+; SI-NEXT: s_and_b32 s49, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s48, s19, 16
+; SI-NEXT: s_and_b32 s51, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s50, s18, 16
+; SI-NEXT: s_and_b32 s53, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s17, 16
+; SI-NEXT: s_and_b32 s55, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s16, 16
+; SI-NEXT: s_and_b32 s65, s5, 0xffff0000
+; SI-NEXT: s_lshl_b32 s64, s5, 16
+; SI-NEXT: s_and_b32 s67, s4, 0xffff0000
+; SI-NEXT: s_lshl_b32 s66, s4, 16
+; SI-NEXT: .LBB61_5: ; %end
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s67
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s66
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s65
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s64
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s55
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s54
+; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s53
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s52
+; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s51
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s50
+; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s49
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s48
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s39
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s38
+; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s37
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s36
+; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s35
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s34
+; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s31
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s30
+; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, vcc_hi
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, vcc_lo
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s95
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s94
+; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s93
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s92
+; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s91
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s90
+; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s89
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s88
+; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s79
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s78
+; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s77
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s76
+; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s75
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s74
+; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s73
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s72
+; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s63
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s62
+; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s61
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s60
+; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s59
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s58
+; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s57
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s56
+; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s47
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s46
+; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s45
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s44
+; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s43
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s42
+; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s41
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s40
+; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s15
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s14
+; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s13
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s12
+; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s11
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
+; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s9
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
+; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
+; SI-NEXT: v_readlane_b32 s4, v34, 0
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
+; SI-NEXT: v_readlane_b32 s4, v34, 1
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
+; SI-NEXT: v_readlane_b32 s30, v33, 34
+; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
+; SI-NEXT: v_readlane_b32 s31, v33, 35
+; SI-NEXT: v_readlane_b32 s99, v33, 33
+; SI-NEXT: v_readlane_b32 s98, v33, 32
+; SI-NEXT: v_readlane_b32 s97, v33, 31
+; SI-NEXT: v_readlane_b32 s96, v33, 30
+; SI-NEXT: v_readlane_b32 s87, v33, 29
+; SI-NEXT: v_readlane_b32 s86, v33, 28
+; SI-NEXT: v_readlane_b32 s85, v33, 27
+; SI-NEXT: v_readlane_b32 s84, v33, 26
+; SI-NEXT: v_readlane_b32 s83, v33, 25
+; SI-NEXT: v_readlane_b32 s82, v33, 24
+; SI-NEXT: v_readlane_b32 s81, v33, 23
+; SI-NEXT: v_readlane_b32 s80, v33, 22
+; SI-NEXT: v_readlane_b32 s71, v33, 21
+; SI-NEXT: v_readlane_b32 s70, v33, 20
+; SI-NEXT: v_readlane_b32 s69, v33, 19
+; SI-NEXT: v_readlane_b32 s68, v33, 18
+; SI-NEXT: v_readlane_b32 s67, v33, 17
+; SI-NEXT: v_readlane_b32 s66, v33, 16
+; SI-NEXT: v_readlane_b32 s65, v33, 15
+; SI-NEXT: v_readlane_b32 s64, v33, 14
+; SI-NEXT: v_readlane_b32 s55, v33, 13
+; SI-NEXT: v_readlane_b32 s54, v33, 12
+; SI-NEXT: v_readlane_b32 s53, v33, 11
+; SI-NEXT: v_readlane_b32 s52, v33, 10
+; SI-NEXT: v_readlane_b32 s51, v33, 9
+; SI-NEXT: v_readlane_b32 s50, v33, 8
+; SI-NEXT: v_readlane_b32 s49, v33, 7
+; SI-NEXT: v_readlane_b32 s48, v33, 6
+; SI-NEXT: v_readlane_b32 s39, v33, 5
+; SI-NEXT: v_readlane_b32 s38, v33, 4
+; SI-NEXT: v_readlane_b32 s37, v33, 3
+; SI-NEXT: v_readlane_b32 s36, v33, 2
+; SI-NEXT: v_readlane_b32 s35, v33, 1
+; SI-NEXT: v_readlane_b32 s34, v33, 0
+; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: s_mov_b64 exec, s[4:5]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v16i64_to_v64bf16_scalar:
; VI: ; %bb.0:
@@ -92687,10 +93283,18 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB61_4
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_3
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s47, s47, 3
; VI-NEXT: s_addc_u32 s46, s46, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -92723,7 +93327,7 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; VI-NEXT: s_addc_u32 s42, s42, 0
; VI-NEXT: s_add_u32 s45, s45, 3
; VI-NEXT: s_addc_u32 s44, s44, 0
-; VI-NEXT: .LBB61_3: ; %end
+; VI-NEXT: .LBB61_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -92757,8 +93361,6 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_4:
-; VI-NEXT: s_branch .LBB61_2
;
; GFX9-LABEL: bitcast_v16i64_to_v64bf16_scalar:
; GFX9: ; %bb.0:
@@ -92783,10 +93385,18 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_3
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s47, s47, 3
; GFX9-NEXT: s_addc_u32 s46, s46, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -92819,7 +93429,7 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX9-NEXT: s_addc_u32 s42, s42, 0
; GFX9-NEXT: s_add_u32 s45, s45, 3
; GFX9-NEXT: s_addc_u32 s44, s44, 0
-; GFX9-NEXT: .LBB61_3: ; %end
+; GFX9-NEXT: .LBB61_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -92853,8 +93463,6 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_4:
-; GFX9-NEXT: s_branch .LBB61_2
;
; GFX11-LABEL: bitcast_v16i64_to_v64bf16_scalar:
; GFX11: ; %bb.0:
@@ -92876,11 +93484,16 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_3
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -92913,7 +93526,7 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX11-NEXT: s_addc_u32 s40, s40, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
-; GFX11-NEXT: .LBB61_3: ; %end
+; GFX11-NEXT: .LBB61_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -92931,8 +93544,6 @@ define inreg <64 x bfloat> @bitcast_v16i64_to_v64bf16_scalar(<16 x i64> inreg %a
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_4:
-; GFX11-NEXT: s_branch .LBB61_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -96449,7 +97060,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_mov_b32_e32 v33, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
@@ -96466,7 +97079,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v40, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_cbranch_vccnz .LBB63_5
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -96836,10 +97449,18 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s50, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -97432,10 +98053,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB63_5
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: s_branch .LBB63_6
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -97468,7 +98087,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB63_5: ; %end
+; VI-NEXT: .LBB63_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v35, 15
; VI-NEXT: v_readlane_b32 s66, v35, 14
; VI-NEXT: v_readlane_b32 s65, v35, 13
@@ -97547,10 +98166,18 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -98161,10 +98788,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: s_branch .LBB63_5
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: s_branch .LBB63_6
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -98197,7 +98822,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB63_5: ; %end
+; GFX9-NEXT: .LBB63_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v36, 15
; GFX9-NEXT: v_readlane_b32 s66, v36, 14
; GFX9-NEXT: v_readlane_b32 s65, v36, 13
@@ -98277,11 +98902,16 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s37, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-TRUE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB63_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s51, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s51, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -98904,8 +99534,6 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v34
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v33.l
; GFX11-TRUE16-NEXT: s_branch .LBB63_5
-; GFX11-TRUE16-NEXT: .LBB63_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB63_2
; GFX11-TRUE16-NEXT: .LBB63_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -99003,11 +99631,16 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s37, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-FAKE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB63_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s51, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s51, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -99661,8 +100294,6 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v32, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v36, 16, v33
; GFX11-FAKE16-NEXT: s_branch .LBB63_5
-; GFX11-FAKE16-NEXT: .LBB63_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB63_2
; GFX11-FAKE16-NEXT: .LBB63_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -100240,7 +100871,7 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB65_4
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 16
; SI-NEXT: s_lshr_b32 s39, s7, 16
@@ -100271,11 +100902,51 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB65_3
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -100337,23 +101008,23 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: .LBB65_3: ; %end
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: .LBB65_5: ; %end
+; SI-NEXT: s_lshl_b32 s47, s34, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: s_lshl_b32 s47, s69, 16
; SI-NEXT: s_or_b32 s17, s17, s47
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s47
; SI-NEXT: s_and_b32 s19, s19, 0xffff
; SI-NEXT: s_lshl_b32 s47, s68, 16
; SI-NEXT: s_or_b32 s19, s19, s47
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s47
; SI-NEXT: s_and_b32 s21, s21, 0xffff
@@ -100496,40 +101167,6 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB65_2
;
; VI-LABEL: bitcast_v16i64_to_v64f16_scalar:
; VI: ; %bb.0:
@@ -100554,10 +101191,18 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB65_4
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_3
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s47, s47, 3
; VI-NEXT: s_addc_u32 s46, s46, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -100590,7 +101235,7 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; VI-NEXT: s_addc_u32 s42, s42, 0
; VI-NEXT: s_add_u32 s45, s45, 3
; VI-NEXT: s_addc_u32 s44, s44, 0
-; VI-NEXT: .LBB65_3: ; %end
+; VI-NEXT: .LBB65_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -100624,8 +101269,6 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_4:
-; VI-NEXT: s_branch .LBB65_2
;
; GFX9-LABEL: bitcast_v16i64_to_v64f16_scalar:
; GFX9: ; %bb.0:
@@ -100650,10 +101293,18 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_3
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s47, s47, 3
; GFX9-NEXT: s_addc_u32 s46, s46, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -100686,7 +101337,7 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX9-NEXT: s_addc_u32 s42, s42, 0
; GFX9-NEXT: s_add_u32 s45, s45, 3
; GFX9-NEXT: s_addc_u32 s44, s44, 0
-; GFX9-NEXT: .LBB65_3: ; %end
+; GFX9-NEXT: .LBB65_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -100720,8 +101371,6 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_4:
-; GFX9-NEXT: s_branch .LBB65_2
;
; GFX11-LABEL: bitcast_v16i64_to_v64f16_scalar:
; GFX11: ; %bb.0:
@@ -100743,11 +101392,16 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_3
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -100780,7 +101434,7 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX11-NEXT: s_addc_u32 s40, s40, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
-; GFX11-NEXT: .LBB65_3: ; %end
+; GFX11-NEXT: .LBB65_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -100798,8 +101452,6 @@ define inreg <64 x half> @bitcast_v16i64_to_v64f16_scalar(<16 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_4:
-; GFX11-NEXT: s_branch .LBB65_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -101822,10 +102474,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v34, s30, 34
; SI-NEXT: v_writelane_b32 v34, s31, 35
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 vcc_hi, s8, 16
; SI-NEXT: v_readfirstlane_b32 s10, v15
; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: v_readfirstlane_b32 s14, v13
@@ -101837,29 +102486,29 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v7
; SI-NEXT: v_readfirstlane_b32 s92, v6
; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s34, v3
-; SI-NEXT: v_readfirstlane_b32 s68, v2
-; SI-NEXT: v_readfirstlane_b32 s80, v1
-; SI-NEXT: v_readfirstlane_b32 s83, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v35, vcc_lo, 0
-; SI-NEXT: s_lshr_b32 s69, s29, 16
-; SI-NEXT: s_lshr_b32 s71, s28, 16
-; SI-NEXT: s_lshr_b32 s82, s27, 16
-; SI-NEXT: s_lshr_b32 s85, s26, 16
-; SI-NEXT: s_lshr_b32 s86, s25, 16
-; SI-NEXT: s_lshr_b32 s87, s24, 16
-; SI-NEXT: s_lshr_b32 s96, s23, 16
-; SI-NEXT: s_lshr_b32 s97, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s7, s19, 16
-; SI-NEXT: s_lshr_b32 s9, s18, 16
-; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: s_lshr_b32 s13, s16, 16
-; SI-NEXT: s_lshr_b32 s65, s10, 16
-; SI-NEXT: s_lshr_b32 s64, s12, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s30, v3
+; SI-NEXT: v_readfirstlane_b32 s34, v2
+; SI-NEXT: v_readfirstlane_b32 s70, v1
+; SI-NEXT: v_readfirstlane_b32 s81, v0
+; SI-NEXT: s_lshr_b32 s35, s29, 16
+; SI-NEXT: s_lshr_b32 s69, s28, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 16
+; SI-NEXT: s_lshr_b32 s83, s26, 16
+; SI-NEXT: s_lshr_b32 s84, s25, 16
+; SI-NEXT: s_lshr_b32 s85, s24, 16
+; SI-NEXT: s_lshr_b32 s86, s23, 16
+; SI-NEXT: s_lshr_b32 s87, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s7, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s6, 16
+; SI-NEXT: s_lshr_b32 s66, s8, 16
+; SI-NEXT: s_lshr_b32 s11, s10, 16
+; SI-NEXT: s_lshr_b32 s13, s12, 16
; SI-NEXT: s_lshr_b32 s15, s14, 16
; SI-NEXT: s_lshr_b32 s73, s72, 16
; SI-NEXT: s_lshr_b32 s75, s74, 16
@@ -101869,74 +102518,75 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s31, s30, 16
-; SI-NEXT: s_lshr_b32 s35, s34, 16
-; SI-NEXT: s_lshr_b32 s70, s68, 16
-; SI-NEXT: s_lshr_b32 s81, s80, 16
-; SI-NEXT: s_lshr_b32 s84, s83, 16
+; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 s71, s70, 16
+; SI-NEXT: s_lshr_b32 s82, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v35, vcc_hi, 1
+; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v35, s65, 2
-; SI-NEXT: v_writelane_b32 v35, s64, 3
-; SI-NEXT: s_cbranch_scc0 .LBB67_3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v35, s67, 0
+; SI-NEXT: v_writelane_b32 v35, s66, 1
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s11, 16
+; SI-NEXT: s_lshl_b32 s41, s7, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
-; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s30, 0xffff
; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -101966,22 +102616,31 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s15, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB67_4
-; SI-NEXT: .LBB67_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s7
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -101993,7 +102652,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s9
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s99
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -102001,11 +102660,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s98
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s99
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s97
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -102016,7 +102675,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s96
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -102025,11 +102684,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s86
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -102040,7 +102699,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s85
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -102049,11 +102708,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s84
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s83
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -102064,7 +102723,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s80
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -102073,11 +102732,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: v_cvt_f32_f16_e32 v13, s28
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s35
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
@@ -102088,8 +102747,8 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v15, s29
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s84
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s81
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -102097,11 +102756,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s71
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s80
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s68
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
@@ -102109,11 +102768,11 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s34
; SI-NEXT: v_or_b32_e32 v15, v16, v15
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
@@ -102121,10 +102780,10 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
; SI-NEXT: v_or_b32_e32 v16, v18, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v18, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v19, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v19, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v20, s95
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
@@ -102195,10 +102854,9 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v25, v27, v25
; SI-NEXT: v_cvt_f32_f16_e32 v27, s15
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_readlane_b32 s4, v35, 3
; SI-NEXT: v_or_b32_e32 v26, v28, v26
; SI-NEXT: v_cvt_f32_f16_e32 v28, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s13
; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v28
@@ -102206,11 +102864,10 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_readlane_b32 s4, v35, 2
; SI-NEXT: v_cvt_f32_f16_e32 v30, s12
; SI-NEXT: v_or_b32_e32 v27, v28, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s11
; SI-NEXT: v_cvt_f32_f16_e32 v31, s10
; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
@@ -102239,11 +102896,8 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v30, v31, v30
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v33, v31
-; SI-NEXT: s_branch .LBB67_5
-; SI-NEXT: .LBB67_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB67_2
-; SI-NEXT: .LBB67_4:
+; SI-NEXT: s_branch .LBB67_6
+; SI-NEXT: .LBB67_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -102276,7 +102930,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB67_5: ; %end
+; SI-NEXT: .LBB67_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v34, 34
; SI-NEXT: v_readlane_b32 s31, v34, 35
; SI-NEXT: v_readlane_b32 s99, v34, 33
@@ -102376,10 +103030,18 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s51, 16
; VI-NEXT: v_mov_b32_e32 v16, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -102541,10 +103203,8 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s52, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB67_5
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: s_branch .LBB67_6
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -102577,7 +103237,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB67_5: ; %end
+; VI-NEXT: .LBB67_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v33, 15
; VI-NEXT: v_readlane_b32 s66, v33, 14
; VI-NEXT: v_readlane_b32 s65, v33, 13
@@ -102656,10 +103316,18 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s51, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s50, v16 op_sel_hi:[1,0]
@@ -102693,10 +103361,8 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v18, s54, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB67_5
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: s_branch .LBB67_6
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -102729,7 +103395,7 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB67_5: ; %end
+; GFX9-NEXT: .LBB67_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -102809,11 +103475,16 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB67_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s51 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s50 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s49 op_sel_hi:[0,1]
@@ -102847,8 +103518,6 @@ define inreg <16 x i64> @bitcast_v64f16_to_v16i64_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s53 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s52 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB67_5
-; GFX11-NEXT: .LBB67_3:
-; GFX11-NEXT: s_branch .LBB67_2
; GFX11-NEXT: .LBB67_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -103426,7 +104095,7 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s45, v1
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: s_cbranch_scc0 .LBB69_4
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 16
; SI-NEXT: s_lshr_b32 s39, s7, 16
@@ -103457,11 +104126,51 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB69_3
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -103523,23 +104232,23 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: .LBB69_3: ; %end
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: .LBB69_5: ; %end
+; SI-NEXT: s_lshl_b32 s47, s34, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s47
; SI-NEXT: s_and_b32 s17, s17, 0xffff
; SI-NEXT: s_lshl_b32 s47, s69, 16
; SI-NEXT: s_or_b32 s17, s17, s47
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s47
; SI-NEXT: s_and_b32 s19, s19, 0xffff
; SI-NEXT: s_lshl_b32 s47, s68, 16
; SI-NEXT: s_or_b32 s19, s19, s47
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s47
; SI-NEXT: s_and_b32 s21, s21, 0xffff
@@ -103682,40 +104391,6 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB69_2
;
; VI-LABEL: bitcast_v16i64_to_v64i16_scalar:
; VI: ; %bb.0:
@@ -103740,10 +104415,18 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB69_4
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB69_3
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s47, s47, 3
; VI-NEXT: s_addc_u32 s46, s46, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -103776,7 +104459,7 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s42, s42, 0
; VI-NEXT: s_add_u32 s45, s45, 3
; VI-NEXT: s_addc_u32 s44, s44, 0
-; VI-NEXT: .LBB69_3: ; %end
+; VI-NEXT: .LBB69_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -103810,8 +104493,6 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_4:
-; VI-NEXT: s_branch .LBB69_2
;
; GFX9-LABEL: bitcast_v16i64_to_v64i16_scalar:
; GFX9: ; %bb.0:
@@ -103836,10 +104517,18 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB69_3
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s47, s47, 3
; GFX9-NEXT: s_addc_u32 s46, s46, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -103872,7 +104561,7 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s42, s42, 0
; GFX9-NEXT: s_add_u32 s45, s45, 3
; GFX9-NEXT: s_addc_u32 s44, s44, 0
-; GFX9-NEXT: .LBB69_3: ; %end
+; GFX9-NEXT: .LBB69_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -103906,8 +104595,6 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v30, s7
; GFX9-NEXT: v_mov_b32_e32 v31, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_4:
-; GFX9-NEXT: s_branch .LBB69_2
;
; GFX11-LABEL: bitcast_v16i64_to_v64i16_scalar:
; GFX11: ; %bb.0:
@@ -103929,11 +104616,16 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_3
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: .LBB69_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -103966,7 +104658,7 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s40, s40, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
-; GFX11-NEXT: .LBB69_3: ; %end
+; GFX11-NEXT: .LBB69_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -103984,8 +104676,6 @@ define inreg <64 x i16> @bitcast_v16i64_to_v64i16_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v28, s7 :: v_dual_mov_b32 v29, s6
; GFX11-NEXT: v_dual_mov_b32 v30, s5 :: v_dual_mov_b32 v31, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_4:
-; GFX11-NEXT: s_branch .LBB69_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -104837,13 +105527,10 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v32, s99, 33
; SI-NEXT: v_writelane_b32 v32, s30, 34
; SI-NEXT: v_writelane_b32 v32, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s9, v16
-; SI-NEXT: s_lshr_b32 s14, s9, 16
-; SI-NEXT: v_readfirstlane_b32 s13, v14
-; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
; SI-NEXT: v_readfirstlane_b32 s11, v15
-; SI-NEXT: s_lshr_b32 s72, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s13, v14
; SI-NEXT: v_readfirstlane_b32 s15, v13
; SI-NEXT: v_readfirstlane_b32 s73, v12
; SI-NEXT: v_readfirstlane_b32 s75, v11
@@ -104852,30 +105539,30 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s89, v8
; SI-NEXT: v_readfirstlane_b32 s91, v7
; SI-NEXT: v_readfirstlane_b32 s94, v6
-; SI-NEXT: v_readfirstlane_b32 s31, v5
-; SI-NEXT: v_readfirstlane_b32 s68, v4
-; SI-NEXT: v_readfirstlane_b32 s80, v3
-; SI-NEXT: v_readfirstlane_b32 s83, v2
-; SI-NEXT: v_readfirstlane_b32 s86, v1
-; SI-NEXT: v_readfirstlane_b32 s97, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; SI-NEXT: v_readfirstlane_b32 s34, v4
+; SI-NEXT: v_readfirstlane_b32 s70, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s84, v1
+; SI-NEXT: v_readfirstlane_b32 s87, v0
; SI-NEXT: s_lshr_b32 s92, s29, 16
; SI-NEXT: s_lshr_b32 s95, s28, 16
-; SI-NEXT: s_lshr_b32 s34, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s71, s25, 16
-; SI-NEXT: s_lshr_b32 s81, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s87, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s6, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s10, s17, 16
-; SI-NEXT: s_lshr_b32 s12, s16, 16
-; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s11, 16
+; SI-NEXT: s_lshr_b32 s30, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s69, s25, 16
+; SI-NEXT: s_lshr_b32 s71, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s85, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s10, s9, 16
+; SI-NEXT: s_lshr_b32 s12, s11, 16
+; SI-NEXT: s_lshr_b32 s14, s13, 16
; SI-NEXT: s_lshr_b32 s63, s15, 16
; SI-NEXT: s_lshr_b32 s62, s73, 16
; SI-NEXT: s_lshr_b32 s61, s75, 16
@@ -104884,54 +105571,55 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s68, 16
-; SI-NEXT: s_lshr_b32 s70, s80, 16
-; SI-NEXT: s_lshr_b32 s82, s83, 16
-; SI-NEXT: s_lshr_b32 s85, s86, 16
-; SI-NEXT: s_lshr_b32 s96, s97, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s70, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v33, s72, 1
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v33, s63, 2
-; SI-NEXT: v_writelane_b32 v33, s62, 3
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_writelane_b32 v33, s63, 1
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s10, 16
+; SI-NEXT: s_lshl_b32 s41, s6, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -104939,23 +105627,23 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s54, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s55, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -104971,92 +105659,109 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s59, s4, s5
; SI-NEXT: s_and_b32 s4, s77, 0xffff
; SI-NEXT: s_lshl_b32 s5, s60, 16
-; SI-NEXT: s_mov_b32 s76, s60
+; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: s_or_b32 s60, s4, s5
; SI-NEXT: s_and_b32 s4, s75, 0xffff
; SI-NEXT: s_lshl_b32 s5, s61, 16
-; SI-NEXT: s_mov_b32 s74, s61
+; SI-NEXT: s_mov_b32 s76, s61
; SI-NEXT: s_or_b32 s61, s4, s5
; SI-NEXT: s_and_b32 s4, s73, 0xffff
; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_mov_b32 s72, s62
; SI-NEXT: s_or_b32 s62, s4, s5
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b32 s74, s60
+; SI-NEXT: s_mov_b32 s76, s61
+; SI-NEXT: s_mov_b32 s72, s62
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_add_i32 s37, s5, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -105068,35 +105773,35 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s97, s97, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
+; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s34, s34, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s54, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s55, s4, 0x30000
@@ -105121,27 +105826,26 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s74, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s75, s75, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s73, s73, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s72, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s15, s15, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 2
+; SI-NEXT: v_readlane_b32 s5, v33, 1
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 1
+; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s63, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s13, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
@@ -105149,21 +105853,20 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_add_i32 s64, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s65, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s66, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s67, s4, 0x30000
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: v_readlane_b32 s30, v32, 34
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
@@ -105238,13 +105941,6 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: s_mov_b32 s76, s60
-; SI-NEXT: s_mov_b32 s74, s61
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v64i16_to_v16i64_scalar:
; VI: ; %bb.0:
@@ -105269,10 +105965,18 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s46, 3
; VI-NEXT: s_and_b32 s4, s46, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -105433,7 +106137,7 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s45, s4, 0x30000
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -105467,8 +106171,6 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v64i16_to_v16i64_scalar:
; GFX9: ; %bb.0:
@@ -105526,10 +106228,18 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB71_4
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -105562,10 +106272,8 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB71_5
-; GFX9-NEXT: .LBB71_3:
-; GFX9-NEXT: s_branch .LBB71_2
-; GFX9-NEXT: .LBB71_4:
+; GFX9-NEXT: s_branch .LBB71_6
+; GFX9-NEXT: .LBB71_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -105598,7 +106306,7 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB71_5: ; %end
+; GFX9-NEXT: .LBB71_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -105678,11 +106386,16 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_4
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB71_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -105716,8 +106429,6 @@ define inreg <16 x i64> @bitcast_v64i16_to_v16i64_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB71_5
-; GFX11-NEXT: .LBB71_3:
-; GFX11-NEXT: s_branch .LBB71_2
; GFX11-NEXT: .LBB71_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -109733,7 +110444,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr61 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB73_3
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s46, s5, 24
; SI-NEXT: v_writelane_b32 v62, s46, 34
@@ -109881,40 +110592,226 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_writelane_b32 v62, s47, 3
; SI-NEXT: s_lshr_b64 s[46:47], s[14:15], 8
; SI-NEXT: v_writelane_b32 v62, s46, 0
-; SI-NEXT: s_lshr_b64 s[48:49], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[4:5], 16
; SI-NEXT: v_writelane_b32 v62, s47, 1
-; SI-NEXT: s_lshr_b64 s[50:51], s[40:41], 24
-; SI-NEXT: s_lshr_b64 s[52:53], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[54:55], s[40:41], 8
-; SI-NEXT: s_lshr_b64 s[64:65], s[42:43], 24
-; SI-NEXT: s_lshr_b64 s[66:67], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[68:69], s[42:43], 8
-; SI-NEXT: s_lshr_b64 s[70:71], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[80:81], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[82:83], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[84:85], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[86:87], s[28:29], 16
-; SI-NEXT: s_lshr_b64 s[96:97], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[98:99], s[26:27], 24
-; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 16
-; SI-NEXT: s_lshr_b64 s[56:57], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[60:61], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[22:23], 8
-; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 24
-; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 8
-; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 24
-; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB73_4
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 s[48:49], s[40:41], 24
+; SI-NEXT: s_lshr_b64 s[50:51], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[40:41], 8
+; SI-NEXT: s_lshr_b64 s[54:55], s[42:43], 24
+; SI-NEXT: s_lshr_b64 s[64:65], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[66:67], s[42:43], 8
+; SI-NEXT: s_lshr_b64 s[68:69], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[70:71], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[44:45], 8
+; SI-NEXT: s_lshr_b64 s[82:83], s[28:29], 24
+; SI-NEXT: s_lshr_b64 s[84:85], s[28:29], 16
+; SI-NEXT: s_lshr_b64 s[86:87], s[28:29], 8
+; SI-NEXT: s_lshr_b64 s[96:97], s[26:27], 24
+; SI-NEXT: s_lshr_b64 s[98:99], s[26:27], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[26:27], 8
+; SI-NEXT: s_lshr_b64 s[56:57], s[24:25], 24
+; SI-NEXT: s_lshr_b64 s[58:59], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[24:25], 8
+; SI-NEXT: s_lshr_b64 s[62:63], s[22:23], 24
+; SI-NEXT: s_lshr_b64 s[72:73], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 8
+; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 24
+; SI-NEXT: s_lshr_b64 s[78:79], s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 8
+; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
+; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 8
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 0
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 2
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 3
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 4
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 5
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 6
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 7
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 9
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 10
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 11
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 13
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 14
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 15
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 16
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 17
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 18
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 19
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 20
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 21
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 22
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 23
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 24
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 25
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 26
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s39, 27
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 28
+; SI-NEXT: v_writelane_b32 v62, s39, 29
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 30
+; SI-NEXT: v_writelane_b32 v62, s39, 31
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: v_writelane_b32 v62, s38, 32
+; SI-NEXT: v_writelane_b32 v62, s39, 33
+; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[1:2], s[4:5], 1.0
; SI-NEXT: v_add_f64 v[3:4], s[6:7], 1.0
; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 24
@@ -110112,213 +111009,32 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b32 s8, s17, 24
; SI-NEXT: s_lshr_b32 vcc_lo, s17, 16
; SI-NEXT: s_lshr_b32 s6, s17, 8
-; SI-NEXT: s_branch .LBB73_5
-; SI-NEXT: .LBB73_3:
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 0
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 1
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 2
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 3
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 4
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 5
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 6
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 7
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 9
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 10
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 11
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 13
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 14
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 15
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 16
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 17
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 18
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 19
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 20
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 21
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 22
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 23
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 24
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 25
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 26
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s49, 27
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 28
-; SI-NEXT: v_writelane_b32 v62, s49, 29
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 30
-; SI-NEXT: v_writelane_b32 v62, s49, 31
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: v_writelane_b32 v62, s48, 32
-; SI-NEXT: v_writelane_b32 v62, s49, 33
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: s_branch .LBB73_2
-; SI-NEXT: .LBB73_4:
+; SI-NEXT: s_branch .LBB73_6
+; SI-NEXT: .LBB73_5:
; SI-NEXT: v_mov_b32_e32 v3, s6
-; SI-NEXT: v_mov_b32_e32 v2, s66
+; SI-NEXT: v_mov_b32_e32 v2, s64
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, s64
-; SI-NEXT: v_mov_b32_e32 v35, s46
+; SI-NEXT: v_mov_b32_e32 v2, s54
+; SI-NEXT: v_mov_b32_e32 v32, s46
; SI-NEXT: v_readlane_b32 s46, v62, 0
; SI-NEXT: v_readlane_b32 s47, v62, 1
-; SI-NEXT: v_mov_b32_e32 v29, s38
-; SI-NEXT: v_mov_b32_e32 v46, s36
-; SI-NEXT: v_mov_b32_e32 v45, s34
-; SI-NEXT: v_mov_b32_e32 v44, s30
-; SI-NEXT: v_mov_b32_e32 v41, s94
-; SI-NEXT: v_mov_b32_e32 v40, s92
-; SI-NEXT: v_mov_b32_e32 v59, s90
-; SI-NEXT: v_mov_b32_e32 v39, s88
-; SI-NEXT: v_mov_b32_e32 v38, s78
-; SI-NEXT: v_mov_b32_e32 v52, s76
-; SI-NEXT: v_mov_b32_e32 v51, s74
-; SI-NEXT: v_mov_b32_e32 v50, s72
-; SI-NEXT: v_mov_b32_e32 v49, s62
-; SI-NEXT: v_mov_b32_e32 v37, s60
-; SI-NEXT: v_mov_b32_e32 v36, s58
-; SI-NEXT: v_mov_b32_e32 v32, s56
+; SI-NEXT: v_mov_b32_e32 v29, s34
+; SI-NEXT: v_mov_b32_e32 v46, s30
+; SI-NEXT: v_mov_b32_e32 v45, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v44, s94
+; SI-NEXT: v_mov_b32_e32 v41, s92
+; SI-NEXT: v_mov_b32_e32 v40, s90
+; SI-NEXT: v_mov_b32_e32 v59, s88
+; SI-NEXT: v_mov_b32_e32 v39, s78
+; SI-NEXT: v_mov_b32_e32 v38, s76
+; SI-NEXT: v_mov_b32_e32 v52, s74
+; SI-NEXT: v_mov_b32_e32 v51, s72
+; SI-NEXT: v_mov_b32_e32 v50, s62
+; SI-NEXT: v_mov_b32_e32 v49, s60
+; SI-NEXT: v_mov_b32_e32 v37, s58
+; SI-NEXT: v_mov_b32_e32 v36, s56
; SI-NEXT: v_mov_b32_e32 v28, s16
; SI-NEXT: v_mov_b32_e32 v20, s18
; SI-NEXT: v_mov_b32_e32 v58, s20
@@ -110334,14 +111050,15 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v7, s10
; SI-NEXT: v_mov_b32_e32 v5, s8
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_mov_b32_e32 v34, s98
-; SI-NEXT: v_mov_b32_e32 v26, s96
-; SI-NEXT: v_mov_b32_e32 v23, s86
-; SI-NEXT: v_mov_b32_e32 v22, s84
-; SI-NEXT: v_mov_b32_e32 v18, s82
-; SI-NEXT: v_mov_b32_e32 v15, s80
-; SI-NEXT: v_mov_b32_e32 v14, s70
-; SI-NEXT: v_mov_b32_e32 v56, s68
+; SI-NEXT: v_mov_b32_e32 v35, s98
+; SI-NEXT: v_mov_b32_e32 v34, s96
+; SI-NEXT: v_mov_b32_e32 v26, s86
+; SI-NEXT: v_mov_b32_e32 v23, s84
+; SI-NEXT: v_mov_b32_e32 v22, s82
+; SI-NEXT: v_mov_b32_e32 v18, s80
+; SI-NEXT: v_mov_b32_e32 v15, s70
+; SI-NEXT: v_mov_b32_e32 v14, s68
+; SI-NEXT: v_mov_b32_e32 v56, s66
; SI-NEXT: v_readlane_b32 s10, v62, 34
; SI-NEXT: v_readlane_b32 s12, v62, 35
; SI-NEXT: v_readlane_b32 s14, v62, 36
@@ -110379,7 +111096,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, s54
+; SI-NEXT: v_mov_b32_e32 v2, s52
; SI-NEXT: v_readlane_b32 s93, v61, 6
; SI-NEXT: v_readlane_b32 s94, v61, 7
; SI-NEXT: v_readlane_b32 s95, v61, 8
@@ -110395,11 +111112,11 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, s52
+; SI-NEXT: v_mov_b32_e32 v2, s50
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, s50
+; SI-NEXT: v_mov_b32_e32 v2, s48
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
@@ -110500,7 +111217,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, s48
+; SI-NEXT: v_mov_b32_e32 v2, s38
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
@@ -110508,7 +111225,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s46, v62, 47
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v28
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v29
@@ -111104,7 +111821,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_cmp_lg_u32 s44, 0
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
; VI-NEXT: v_writelane_b32 v62, s46, 10
@@ -111255,13 +111972,175 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[18:19], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[16:17], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s47, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s47, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[13:14], s[4:5], 1.0
; VI-NEXT: v_add_f64 v[1:2], s[6:7], 1.0
; VI-NEXT: v_add_f64 v[3:4], s[8:9], 1.0
@@ -111457,169 +112336,13 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v36, 8, v32
; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v31
; VI-NEXT: v_lshrrev_b32_e32 v46, 8, v31
-; VI-NEXT: s_branch .LBB73_5
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
-; VI-NEXT: v_mov_b32_e32 v42, s48
-; VI-NEXT: v_mov_b32_e32 v41, s38
+; VI-NEXT: s_branch .LBB73_6
+; VI-NEXT: .LBB73_5:
+; VI-NEXT: v_mov_b32_e32 v42, s38
+; VI-NEXT: v_mov_b32_e32 v41, s36
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v41, s36
+; VI-NEXT: v_mov_b32_e32 v41, s34
; VI-NEXT: v_mov_b32_e32 v34, s65
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v34, s64
@@ -111698,7 +112421,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v41, s34
+; VI-NEXT: v_mov_b32_e32 v41, s30
; VI-NEXT: v_mov_b32_e32 v34, s4
; VI-NEXT: v_readlane_b32 s4, v62, 15
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
@@ -111777,7 +112500,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readlane_b32 s4, v62, 41
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v41, s30
+; VI-NEXT: v_mov_b32_e32 v41, vcc_lo
; VI-NEXT: v_mov_b32_e32 v38, s4
; VI-NEXT: v_readlane_b32 s4, v62, 42
; VI-NEXT: v_mov_b32_e32 v47, s4
@@ -111884,7 +112607,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v41, s46
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; VI-NEXT: .LBB73_5: ; %end
+; VI-NEXT: .LBB73_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v41, v45, v42, s4
; VI-NEXT: v_perm_b32 v31, v31, v46, s4
@@ -112336,7 +113059,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_cmp_lg_u32 s44, 0
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
; GFX9-NEXT: v_writelane_b32 v62, s46, 2
@@ -112481,11 +113204,165 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
; GFX9-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[11:12], s[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[1:2], s[6:7], 1.0
; GFX9-NEXT: v_add_f64 v[3:4], s[8:9], 1.0
@@ -112700,156 +113577,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 8, v49
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 16, v48
; GFX9-NEXT: v_lshrrev_b32_e32 v60, 8, v48
-; GFX9-NEXT: s_branch .LBB73_5
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: s_branch .LBB73_6
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v15, s66
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s65
@@ -112987,11 +113716,11 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 31
; GFX9-NEXT: v_mov_b32_e32 v58, s4
-; GFX9-NEXT: v_mov_b32_e32 v57, s34
+; GFX9-NEXT: v_mov_b32_e32 v57, s30
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v57, s30
+; GFX9-NEXT: v_mov_b32_e32 v57, vcc_lo
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
@@ -113003,7 +113732,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v52, s36
+; GFX9-NEXT: v_mov_b32_e32 v52, s34
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
@@ -113131,7 +113860,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s67
; GFX9-NEXT: v_mov_b32_e32 v20, s4
; GFX9-NEXT: v_mov_b32_e32 v51, s55
-; GFX9-NEXT: .LBB73_5: ; %end
+; GFX9-NEXT: .LBB73_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v39, v39, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
@@ -113591,10 +114320,10 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-NEXT: v_readfirstlane_b32 s12, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s30, 0
; GFX11-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB73_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s11, 16
; GFX11-NEXT: s_lshr_b32 s48, s41, 24
@@ -113661,7 +114390,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_writelane_b32 v76, s42, 12
; GFX11-NEXT: s_lshr_b32 s42, s29, 24
; GFX11-NEXT: s_lshr_b32 s101, s2, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s0, 16
+; GFX11-NEXT: s_lshr_b32 s31, s0, 16
; GFX11-NEXT: s_lshr_b32 s103, s0, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 13
; GFX11-NEXT: s_lshr_b32 s42, s29, 16
@@ -113677,7 +114406,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: s_lshr_b32 s42, s27, 24
; GFX11-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
; GFX11-NEXT: s_lshr_b64 s[94:95], s[28:29], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[26:27], 24
+; GFX11-NEXT: s_lshr_b64 vcc, s[26:27], 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 16
; GFX11-NEXT: s_lshr_b32 s42, s27, 16
; GFX11-NEXT: s_lshr_b64 s[62:63], s[24:25], 24
@@ -113743,129 +114472,15 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_writelane_b32 v77, s42, 7
; GFX11-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
-; GFX11-NEXT: v_add_f64 v[21:22], s[22:23], 1.0
-; GFX11-NEXT: v_add_f64 v[11:12], s[4:5], 1.0
-; GFX11-NEXT: v_add_f64 v[23:24], s[20:21], 1.0
-; GFX11-NEXT: v_add_f64 v[13:14], s[12:13], 1.0
-; GFX11-NEXT: v_add_f64 v[27:28], s[18:19], 1.0
-; GFX11-NEXT: v_add_f64 v[15:16], s[28:29], 1.0
-; GFX11-NEXT: v_add_f64 v[31:32], s[16:17], 1.0
-; GFX11-NEXT: v_add_f64 v[7:8], s[8:9], 1.0
-; GFX11-NEXT: v_add_f64 v[17:18], s[26:27], 1.0
-; GFX11-NEXT: v_add_f64 v[35:36], s[2:3], 1.0
-; GFX11-NEXT: v_add_f64 v[1:2], s[40:41], 1.0
-; GFX11-NEXT: v_add_f64 v[3:4], s[14:15], 1.0
-; GFX11-NEXT: v_add_f64 v[5:6], s[10:11], 1.0
-; GFX11-NEXT: v_add_f64 v[9:10], s[6:7], 1.0
-; GFX11-NEXT: v_add_f64 v[19:20], s[24:25], 1.0
-; GFX11-NEXT: v_add_f64 v[48:49], s[0:1], 1.0
-; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[21:22]
-; GFX11-NEXT: v_lshrrev_b64 v[50:51], 24, v[11:12]
-; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[23:24]
-; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[13:14]
-; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[27:28]
-; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[15:16]
-; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[31:32]
-; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[7:8]
-; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-NEXT: v_lshrrev_b64 v[68:69], 24, v[35:36]
-; GFX11-NEXT: v_lshrrev_b64 v[25:26], 24, v[1:2]
-; GFX11-NEXT: v_lshrrev_b64 v[29:30], 24, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[33:34], 24, v[5:6]
-; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[9:10]
-; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[19:20]
-; GFX11-NEXT: v_lshrrev_b64 v[69:70], 24, v[48:49]
-; GFX11-NEXT: v_lshrrev_b32_e32 v39, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v55, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v34, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v82, 24, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v83, 16, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v81, 8, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v71, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v97, 24, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v98, 16, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v86, 8, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v84, 8, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v102, 24, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v101, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v96, 16, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v114, 24, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v116, 16, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v115, 8, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v100, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v99, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v131, 16, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v129, 8, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v113, 16, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v112, 8, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v134, 24, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v144, 8, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v118, 16, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v117, 8, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v150, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v147, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v128, 16, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v119, 8, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v164, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v163, 8, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v133, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v176, 24, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v146, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v135, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v182, 24, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v40, 16, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v151, 16, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v44, 24, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v45, 16, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v43, 8, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v160, 8, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v46, 24, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v56, 16, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v47, 8, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v166, 16, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v165, 8, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v58, 24, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v59, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v178, 16, v31
-; GFX11-NEXT: v_lshrrev_b32_e32 v177, 8, v31
-; GFX11-NEXT: v_lshrrev_b32_e32 v60, 24, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v62, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v61, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v180, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v63, 24, v49
-; GFX11-NEXT: v_lshrrev_b32_e32 v72, 16, v49
-; GFX11-NEXT: v_lshrrev_b32_e32 v73, 8, v49
-; GFX11-NEXT: v_lshrrev_b32_e32 v41, 16, v48
-; GFX11-NEXT: v_lshrrev_b32_e32 v42, 8, v48
-; GFX11-NEXT: s_branch .LBB73_5
-; GFX11-NEXT: .LBB73_3:
+; GFX11-NEXT: s_branch .LBB73_3
+; GFX11-NEXT: .LBB73_2:
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: s_mov_b32 s30, -1
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr103
-; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $sgpr31
; GFX11-NEXT: ; implicit-def: $sgpr42
; GFX11-NEXT: ; implicit-def: $sgpr101
; GFX11-NEXT: ; implicit-def: $sgpr102
@@ -113911,7 +114526,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: ; implicit-def: $sgpr38
; GFX11-NEXT: ; implicit-def: $sgpr39
; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $vcc_lo
; GFX11-NEXT: ; implicit-def: $sgpr94
; GFX11-NEXT: ; implicit-def: $sgpr92
; GFX11-NEXT: ; implicit-def: $sgpr90
@@ -113996,8 +114611,127 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: s_branch .LBB73_2
-; GFX11-NEXT: .LBB73_4:
+; GFX11-NEXT: .LBB73_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s43, s30, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_add_f64 v[21:22], s[22:23], 1.0
+; GFX11-NEXT: v_add_f64 v[11:12], s[4:5], 1.0
+; GFX11-NEXT: v_add_f64 v[23:24], s[20:21], 1.0
+; GFX11-NEXT: v_add_f64 v[13:14], s[12:13], 1.0
+; GFX11-NEXT: v_add_f64 v[27:28], s[18:19], 1.0
+; GFX11-NEXT: v_add_f64 v[15:16], s[28:29], 1.0
+; GFX11-NEXT: v_add_f64 v[31:32], s[16:17], 1.0
+; GFX11-NEXT: v_add_f64 v[7:8], s[8:9], 1.0
+; GFX11-NEXT: v_add_f64 v[17:18], s[26:27], 1.0
+; GFX11-NEXT: v_add_f64 v[35:36], s[2:3], 1.0
+; GFX11-NEXT: v_add_f64 v[1:2], s[40:41], 1.0
+; GFX11-NEXT: v_add_f64 v[3:4], s[14:15], 1.0
+; GFX11-NEXT: v_add_f64 v[5:6], s[10:11], 1.0
+; GFX11-NEXT: v_add_f64 v[9:10], s[6:7], 1.0
+; GFX11-NEXT: v_add_f64 v[19:20], s[24:25], 1.0
+; GFX11-NEXT: v_add_f64 v[48:49], s[0:1], 1.0
+; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[21:22]
+; GFX11-NEXT: v_lshrrev_b64 v[50:51], 24, v[11:12]
+; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[23:24]
+; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[13:14]
+; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[27:28]
+; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[15:16]
+; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[31:32]
+; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[7:8]
+; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18]
+; GFX11-NEXT: v_lshrrev_b64 v[68:69], 24, v[35:36]
+; GFX11-NEXT: v_lshrrev_b64 v[25:26], 24, v[1:2]
+; GFX11-NEXT: v_lshrrev_b64 v[29:30], 24, v[3:4]
+; GFX11-NEXT: v_lshrrev_b64 v[33:34], 24, v[5:6]
+; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[9:10]
+; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-NEXT: v_lshrrev_b64 v[69:70], 24, v[48:49]
+; GFX11-NEXT: v_lshrrev_b32_e32 v39, 24, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v34, 8, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v26, 8, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v82, 24, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v83, 16, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v81, 8, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v71, 8, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v97, 24, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v98, 16, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v86, 8, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v84, 8, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v102, 24, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v101, 8, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v96, 16, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v114, 24, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v116, 16, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v115, 8, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v100, 16, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v99, 8, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v131, 16, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v129, 8, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v113, 16, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v112, 8, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v134, 24, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v144, 8, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v118, 16, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v117, 8, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v150, 16, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v147, 8, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v128, 16, v15
+; GFX11-NEXT: v_lshrrev_b32_e32 v119, 8, v15
+; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v164, 16, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v163, 8, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v133, 16, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v176, 24, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v146, 16, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v135, 8, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v182, 24, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v40, 16, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v151, 16, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v44, 24, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v45, 16, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v43, 8, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v160, 8, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v46, 24, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v56, 16, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v47, 8, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v166, 16, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v165, 8, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v58, 24, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v59, 16, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v178, 16, v31
+; GFX11-NEXT: v_lshrrev_b32_e32 v177, 8, v31
+; GFX11-NEXT: v_lshrrev_b32_e32 v60, 24, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v62, 16, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v61, 8, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v180, 8, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v63, 24, v49
+; GFX11-NEXT: v_lshrrev_b32_e32 v72, 16, v49
+; GFX11-NEXT: v_lshrrev_b32_e32 v73, 8, v49
+; GFX11-NEXT: v_lshrrev_b32_e32 v41, 16, v48
+; GFX11-NEXT: v_lshrrev_b32_e32 v42, 8, v48
+; GFX11-NEXT: s_branch .LBB73_6
+; GFX11-NEXT: .LBB73_5:
; GFX11-NEXT: v_dual_mov_b32 v48, s0 :: v_dual_mov_b32 v49, s1
; GFX11-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-NEXT: v_mov_b32_e32 v83, s36
@@ -114035,7 +114769,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_mov_b32_e32 v115, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 7
; GFX11-NEXT: v_dual_mov_b32 v35, s2 :: v_dual_mov_b32 v36, s3
-; GFX11-NEXT: v_dual_mov_b32 v41, vcc_hi :: v_dual_mov_b32 v42, s103
+; GFX11-NEXT: v_dual_mov_b32 v41, s31 :: v_dual_mov_b32 v42, s103
; GFX11-NEXT: v_mov_b32_e32 v130, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 8
; GFX11-NEXT: v_dual_mov_b32 v181, s102 :: v_dual_mov_b32 v180, s101
@@ -114084,7 +114818,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_mov_b32_e32 v164, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 18
; GFX11-NEXT: v_dual_mov_b32 v65, s58 :: v_dual_mov_b32 v64, s60
-; GFX11-NEXT: v_dual_mov_b32 v54, s62 :: v_dual_mov_b32 v53, s30
+; GFX11-NEXT: v_dual_mov_b32 v54, s62 :: v_dual_mov_b32 v53, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v163, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 19
@@ -114144,7 +114878,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_mov_b32_e32 v72, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v73, s0
-; GFX11-NEXT: .LBB73_5: ; %end
+; GFX11-NEXT: .LBB73_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v69, v41, v69, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v48, v48, v42, 0xc0c0004
@@ -120868,8 +121602,10 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_cbranch_vccnz .LBB75_5
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
@@ -121866,8 +122602,10 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; VI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; VI-NEXT: s_cbranch_vccnz .LBB75_5
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
@@ -122790,8 +123528,10 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB75_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
@@ -123286,7 +124026,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB75_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB75_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
@@ -123432,9 +124172,17 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB75_3
-; GFX11-TRUE16-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB75_3
+; GFX11-TRUE16-NEXT: .LBB75_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB75_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
@@ -123725,15 +124473,12 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-TRUE16-NEXT: .LBB75_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB75_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB75_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB75_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v16f64_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -123860,7 +124605,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB75_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB75_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -124006,9 +124751,17 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v31, v183
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v40, v41
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB75_3
-; GFX11-FAKE16-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB75_3
+; GFX11-FAKE16-NEXT: .LBB75_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB75_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
@@ -124299,15 +125052,12 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_or_b32_e32 v29, v31, v34
; GFX11-FAKE16-NEXT: v_or_b32_e32 v30, v36, v35
; GFX11-FAKE16-NEXT: v_or_b32_e32 v31, v33, v32
-; GFX11-FAKE16-NEXT: .LBB75_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB75_5: ; %end
; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB75_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB75_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -125047,8 +125797,8 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -125121,55 +125871,51 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: s_cmp_lg_u32 s44, 0
; SI-NEXT: v_readfirstlane_b32 s44, v0
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB77_3
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s46, s17, 0xffff0000
-; SI-NEXT: v_writelane_b32 v62, s46, 0
-; SI-NEXT: s_lshl_b32 s46, s17, 16
-; SI-NEXT: v_writelane_b32 v62, s46, 1
; SI-NEXT: s_and_b32 s46, s16, 0xffff0000
-; SI-NEXT: v_writelane_b32 v62, s46, 2
+; SI-NEXT: v_writelane_b32 v62, s46, 0
; SI-NEXT: s_lshl_b32 s46, s16, 16
-; SI-NEXT: s_and_b32 s59, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s5, 16
-; SI-NEXT: s_and_b32 s57, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s4, 16
-; SI-NEXT: s_and_b32 s99, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s98, s7, 16
-; SI-NEXT: s_and_b32 s97, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s96, s6, 16
-; SI-NEXT: s_and_b32 s87, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s86, s9, 16
-; SI-NEXT: s_and_b32 s85, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s84, s8, 16
-; SI-NEXT: s_and_b32 s83, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s82, s11, 16
-; SI-NEXT: s_and_b32 s81, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s80, s10, 16
-; SI-NEXT: s_and_b32 s71, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s70, s13, 16
-; SI-NEXT: s_and_b32 s69, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s68, s12, 16
-; SI-NEXT: s_and_b32 s67, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s66, s15, 16
-; SI-NEXT: s_and_b32 s65, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s64, s14, 16
-; SI-NEXT: s_and_b32 s55, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s41, 16
-; SI-NEXT: s_and_b32 s53, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s52, s40, 16
-; SI-NEXT: s_and_b32 s51, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s43, 16
-; SI-NEXT: s_and_b32 s49, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s48, s42, 16
-; SI-NEXT: s_and_b32 s39, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, s45, 16
-; SI-NEXT: s_and_b32 s37, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s36, s44, 16
-; SI-NEXT: s_and_b32 s35, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s29, 16
-; SI-NEXT: s_and_b32 s31, s28, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s28, 16
+; SI-NEXT: s_and_b32 s57, s5, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s5, 16
+; SI-NEXT: s_and_b32 s99, s4, 0xffff0000
+; SI-NEXT: s_lshl_b32 s98, s4, 16
+; SI-NEXT: s_and_b32 s97, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s96, s7, 16
+; SI-NEXT: s_and_b32 s87, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s86, s6, 16
+; SI-NEXT: s_and_b32 s85, s9, 0xffff0000
+; SI-NEXT: s_lshl_b32 s84, s9, 16
+; SI-NEXT: s_and_b32 s83, s8, 0xffff0000
+; SI-NEXT: s_lshl_b32 s82, s8, 16
+; SI-NEXT: s_and_b32 s81, s11, 0xffff0000
+; SI-NEXT: s_lshl_b32 s80, s11, 16
+; SI-NEXT: s_and_b32 s71, s10, 0xffff0000
+; SI-NEXT: s_lshl_b32 s70, s10, 16
+; SI-NEXT: s_and_b32 s69, s13, 0xffff0000
+; SI-NEXT: s_lshl_b32 s68, s13, 16
+; SI-NEXT: s_and_b32 s67, s12, 0xffff0000
+; SI-NEXT: s_lshl_b32 s66, s12, 16
+; SI-NEXT: s_and_b32 s65, s15, 0xffff0000
+; SI-NEXT: s_lshl_b32 s64, s15, 16
+; SI-NEXT: s_and_b32 s55, s14, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s14, 16
+; SI-NEXT: s_and_b32 s53, s41, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s41, 16
+; SI-NEXT: s_and_b32 s51, s40, 0xffff0000
+; SI-NEXT: s_lshl_b32 s50, s40, 16
+; SI-NEXT: s_and_b32 s49, s43, 0xffff0000
+; SI-NEXT: s_lshl_b32 s48, s43, 16
+; SI-NEXT: s_and_b32 s39, s42, 0xffff0000
+; SI-NEXT: s_lshl_b32 s38, s42, 16
+; SI-NEXT: s_and_b32 s37, s45, 0xffff0000
+; SI-NEXT: s_lshl_b32 s36, s45, 16
+; SI-NEXT: s_and_b32 s35, s44, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s44, 16
+; SI-NEXT: s_and_b32 s31, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s30, s29, 16
+; SI-NEXT: s_and_b32 vcc_hi, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, s28, 16
; SI-NEXT: s_and_b32 s95, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s27, 16
; SI-NEXT: s_and_b32 s93, s26, 0xffff0000
@@ -125190,110 +125936,16 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: s_lshl_b32 s62, s19, 16
; SI-NEXT: s_and_b32 s61, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s60, s18, 16
-; SI-NEXT: v_writelane_b32 v62, s46, 3
-; SI-NEXT: s_cbranch_execnz .LBB77_4
-; SI-NEXT: .LBB77_2: ; %cmp.true
-; SI-NEXT: v_add_f64 v[22:23], s[4:5], 1.0
-; SI-NEXT: v_add_f64 v[0:1], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[2:3], s[24:25], 1.0
-; SI-NEXT: v_add_f64 v[14:15], s[14:15], 1.0
-; SI-NEXT: v_add_f64 v[16:17], s[12:13], 1.0
-; SI-NEXT: v_add_f64 v[24:25], s[6:7], 1.0
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v23
-; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; SI-NEXT: v_add_f64 v[10:11], s[42:43], 1.0
-; SI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v17
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v17
-; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v15
-; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v15
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v2
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; SI-NEXT: v_add_f64 v[1:2], s[20:21], 1.0
-; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v25
-; SI-NEXT: v_add_f64 v[4:5], s[26:27], 1.0
-; SI-NEXT: v_add_f64 v[6:7], s[28:29], 1.0
-; SI-NEXT: v_add_f64 v[8:9], s[44:45], 1.0
-; SI-NEXT: v_add_f64 v[12:13], s[40:41], 1.0
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
-; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v19
-; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v18
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v11
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v10
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; SI-NEXT: v_add_f64 v[37:38], s[16:17], 1.0
-; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v25
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v24
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v21
-; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v20
-; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v16
-; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v13
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v13
-; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v12
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v9
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v9
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v8
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v7
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v7
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v5
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v5
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v0
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v38
-; SI-NEXT: v_mov_b32_e32 v38, v34
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v37
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v37
-; SI-NEXT: s_branch .LBB77_5
-; SI-NEXT: .LBB77_3:
+; SI-NEXT: s_and_b32 s59, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s58, s17, 16
+; SI-NEXT: v_writelane_b32 v62, s46, 1
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; kill: killed $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr61
; SI-NEXT: ; implicit-def: $sgpr62
@@ -125314,6 +125966,8 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr34
@@ -125352,79 +126006,169 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: ; implicit-def: $sgpr99
; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; kill: killed $sgpr46
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; kill: killed $sgpr46
-; SI-NEXT: s_branch .LBB77_2
-; SI-NEXT: .LBB77_4:
-; SI-NEXT: v_mov_b32_e32 v0, s59
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s46, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s46, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f64 v[22:23], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[0:1], s[22:23], 1.0
+; SI-NEXT: v_add_f64 v[2:3], s[24:25], 1.0
+; SI-NEXT: v_add_f64 v[14:15], s[14:15], 1.0
+; SI-NEXT: v_add_f64 v[16:17], s[12:13], 1.0
+; SI-NEXT: v_add_f64 v[24:25], s[6:7], 1.0
+; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v23
+; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s58
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_add_f64 v[10:11], s[42:43], 1.0
+; SI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v25
+; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v17
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v17
+; SI-NEXT: v_and_b32_e32 v47, 0xffff0000, v15
+; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v15
+; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v14
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v2
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; SI-NEXT: v_add_f64 v[1:2], s[20:21], 1.0
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; SI-NEXT: v_add_f64 v[4:5], s[26:27], 1.0
+; SI-NEXT: v_add_f64 v[12:13], s[40:41], 1.0
+; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v55, 0xffff0000, v19
+; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v19
+; SI-NEXT: v_and_b32_e32 v41, 0xffff0000, v18
+; SI-NEXT: v_lshlrev_b32_e32 v40, 16, v18
+; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v11
+; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v11
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; SI-NEXT: v_add_f64 v[6:7], s[28:29], 1.0
+; SI-NEXT: v_add_f64 v[8:9], s[44:45], 1.0
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v21
+; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v21
+; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v20
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v20
+; SI-NEXT: v_and_b32_e32 v59, 0xffff0000, v13
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v13
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v5
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v5
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; SI-NEXT: v_add_f64 v[1:2], s[16:17], 1.0
+; SI-NEXT: v_and_b32_e32 v45, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v16
+; SI-NEXT: v_and_b32_e32 v61, 0xffff0000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v60, 16, v12
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v9
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v9
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v8
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v7
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v7
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v6
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v0
+; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v3
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v34
+; SI-NEXT: s_branch .LBB77_6
+; SI-NEXT: .LBB77_5:
; SI-NEXT: v_mov_b32_e32 v0, s56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s99
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s98
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s97
-; SI-NEXT: v_readlane_b32 s4, v62, 0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s96
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s87
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s86
-; SI-NEXT: v_mov_b32_e32 v9, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 3
-; SI-NEXT: v_mov_b32_e32 v38, s98
-; SI-NEXT: v_mov_b32_e32 v51, s87
-; SI-NEXT: v_mov_b32_e32 v53, s85
-; SI-NEXT: v_mov_b32_e32 v52, s84
-; SI-NEXT: v_mov_b32_e32 v55, s83
-; SI-NEXT: v_mov_b32_e32 v54, s82
-; SI-NEXT: v_mov_b32_e32 v41, s81
-; SI-NEXT: v_mov_b32_e32 v40, s80
-; SI-NEXT: v_mov_b32_e32 v43, s71
-; SI-NEXT: v_mov_b32_e32 v42, s70
-; SI-NEXT: v_mov_b32_e32 v45, s69
-; SI-NEXT: v_mov_b32_e32 v44, s68
-; SI-NEXT: v_mov_b32_e32 v47, s67
-; SI-NEXT: v_mov_b32_e32 v46, s66
-; SI-NEXT: v_mov_b32_e32 v57, s65
-; SI-NEXT: v_mov_b32_e32 v56, s64
-; SI-NEXT: v_mov_b32_e32 v59, s55
-; SI-NEXT: v_mov_b32_e32 v58, s54
-; SI-NEXT: v_mov_b32_e32 v61, s53
-; SI-NEXT: v_mov_b32_e32 v60, s52
-; SI-NEXT: v_mov_b32_e32 v48, s51
-; SI-NEXT: v_mov_b32_e32 v39, s50
-; SI-NEXT: v_mov_b32_e32 v33, s49
-; SI-NEXT: v_mov_b32_e32 v32, s48
-; SI-NEXT: v_mov_b32_e32 v31, s39
-; SI-NEXT: v_mov_b32_e32 v30, s38
-; SI-NEXT: v_mov_b32_e32 v29, s37
-; SI-NEXT: v_mov_b32_e32 v28, s36
-; SI-NEXT: v_mov_b32_e32 v27, s35
-; SI-NEXT: v_mov_b32_e32 v26, s34
-; SI-NEXT: v_mov_b32_e32 v25, s31
-; SI-NEXT: v_mov_b32_e32 v24, s30
+; SI-NEXT: v_readlane_b32 s4, v62, 1
+; SI-NEXT: v_mov_b32_e32 v1, s57
+; SI-NEXT: v_mov_b32_e32 v51, s85
+; SI-NEXT: v_mov_b32_e32 v50, s84
+; SI-NEXT: v_mov_b32_e32 v53, s83
+; SI-NEXT: v_mov_b32_e32 v52, s82
+; SI-NEXT: v_mov_b32_e32 v55, s81
+; SI-NEXT: v_mov_b32_e32 v54, s80
+; SI-NEXT: v_mov_b32_e32 v41, s71
+; SI-NEXT: v_mov_b32_e32 v40, s70
+; SI-NEXT: v_mov_b32_e32 v43, s69
+; SI-NEXT: v_mov_b32_e32 v42, s68
+; SI-NEXT: v_mov_b32_e32 v45, s67
+; SI-NEXT: v_mov_b32_e32 v44, s66
+; SI-NEXT: v_mov_b32_e32 v47, s65
+; SI-NEXT: v_mov_b32_e32 v46, s64
+; SI-NEXT: v_mov_b32_e32 v57, s55
+; SI-NEXT: v_mov_b32_e32 v56, s54
+; SI-NEXT: v_mov_b32_e32 v59, s53
+; SI-NEXT: v_mov_b32_e32 v58, s52
+; SI-NEXT: v_mov_b32_e32 v61, s51
+; SI-NEXT: v_mov_b32_e32 v60, s50
+; SI-NEXT: v_mov_b32_e32 v38, s49
+; SI-NEXT: v_mov_b32_e32 v37, s48
+; SI-NEXT: v_mov_b32_e32 v33, s39
+; SI-NEXT: v_mov_b32_e32 v32, s38
+; SI-NEXT: v_mov_b32_e32 v31, s37
+; SI-NEXT: v_mov_b32_e32 v30, s36
+; SI-NEXT: v_mov_b32_e32 v29, s35
+; SI-NEXT: v_mov_b32_e32 v28, s34
+; SI-NEXT: v_mov_b32_e32 v27, s31
+; SI-NEXT: v_mov_b32_e32 v26, s30
+; SI-NEXT: v_mov_b32_e32 v25, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v24, vcc_lo
; SI-NEXT: v_mov_b32_e32 v23, s95
; SI-NEXT: v_mov_b32_e32 v22, s94
; SI-NEXT: v_mov_b32_e32 v21, s93
@@ -125432,53 +126176,51 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_mov_b32_e32 v19, s91
; SI-NEXT: v_mov_b32_e32 v18, s90
; SI-NEXT: v_mov_b32_e32 v17, s89
-; SI-NEXT: v_mov_b32_e32 v50, s88
+; SI-NEXT: v_mov_b32_e32 v49, s88
; SI-NEXT: v_mov_b32_e32 v15, s79
; SI-NEXT: v_mov_b32_e32 v14, s78
; SI-NEXT: v_mov_b32_e32 v12, s77
; SI-NEXT: v_mov_b32_e32 v36, s76
; SI-NEXT: v_mov_b32_e32 v11, s75
; SI-NEXT: v_mov_b32_e32 v10, s74
-; SI-NEXT: v_mov_b32_e32 v49, s73
+; SI-NEXT: v_mov_b32_e32 v39, s73
; SI-NEXT: v_mov_b32_e32 v13, s72
-; SI-NEXT: v_mov_b32_e32 v1, s63
+; SI-NEXT: v_mov_b32_e32 v48, s63
; SI-NEXT: v_mov_b32_e32 v6, s62
; SI-NEXT: v_mov_b32_e32 v5, s61
; SI-NEXT: v_mov_b32_e32 v4, s60
-; SI-NEXT: v_mov_b32_e32 v3, s4
-; SI-NEXT: .LBB77_5: ; %end
+; SI-NEXT: v_mov_b32_e32 v3, s59
+; SI-NEXT: v_mov_b32_e32 v2, s58
+; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: .LBB77_6: ; %end
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v2
-; SI-NEXT: v_mul_f32_e32 v34, 1.0, v3
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v49
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v48
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v39
+; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; SI-NEXT: v_mul_f32_e32 v4, 1.0, v6
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v11
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v10
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; SI-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v12
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v36
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e32 v34, 1.0, v9
; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
+; SI-NEXT: v_mul_f32_e32 v9, 1.0, v36
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v17
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v19
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v0
@@ -125496,14 +126238,17 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v33
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: v_mul_f32_e32 v11, 1.0, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v48
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v38
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v13
; SI-NEXT: v_mul_f32_e32 v13, 1.0, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v61
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v59
; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v57
@@ -125522,23 +126267,22 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v2, 1.0, v9
-; SI-NEXT: v_mul_f32_e32 v9, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: v_mul_f32_e32 v9, 1.0, v49
+; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_lshr_b64 v[33:34], v[34:35], 16
; SI-NEXT: v_lshr_b64 v[34:35], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e32 v19, 1.0, v39
-; SI-NEXT: v_mul_f32_e32 v36, 1.0, v60
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_mul_f32_e32 v19, 1.0, v37
; SI-NEXT: v_mul_f32_e32 v22, 1.0, v22
; SI-NEXT: v_mul_f32_e32 v24, 1.0, v24
; SI-NEXT: v_mul_f32_e32 v26, 1.0, v26
; SI-NEXT: v_mul_f32_e32 v28, 1.0, v28
; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
; SI-NEXT: v_mul_f32_e32 v17, 1.0, v32
+; SI-NEXT: v_mul_f32_e32 v60, 1.0, v60
; SI-NEXT: v_mul_f32_e32 v58, 1.0, v58
; SI-NEXT: v_mul_f32_e32 v56, 1.0, v56
; SI-NEXT: v_mul_f32_e32 v46, 1.0, v46
@@ -125547,7 +126291,7 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_mul_f32_e32 v40, 1.0, v40
; SI-NEXT: v_mul_f32_e32 v54, 1.0, v54
; SI-NEXT: v_mul_f32_e32 v52, 1.0, v52
-; SI-NEXT: v_mul_f32_e32 v38, 1.0, v38
+; SI-NEXT: v_mul_f32_e32 v50, 1.0, v50
; SI-NEXT: v_readlane_b32 s30, v63, 34
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s99, v63, 33
@@ -125585,25 +126329,26 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[4:5], v[5:6], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[5:6], 16
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_mul_f32_e32 v48, 1.0, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
@@ -125616,30 +126361,13 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: v_lshr_b64 v[15:16], v[30:31], 16
; SI-NEXT: v_lshr_b64 v[16:17], v[17:18], 16
; SI-NEXT: v_lshr_b64 v[17:18], v[19:20], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
; SI-NEXT: v_lshr_b64 v[19:20], v[58:59], 16
; SI-NEXT: v_lshr_b64 v[20:21], v[56:57], 16
; SI-NEXT: v_lshr_b64 v[21:22], v[46:47], 16
; SI-NEXT: v_lshr_b64 v[22:23], v[44:45], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[42:43], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[40:41], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[48:49], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[29:30], v[38:39], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mul_f32_e32 v60, 1.0, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[30:31], v[60:61], 16
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -125654,19 +126382,32 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[25:26], v[54:55], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
+; SI-NEXT: v_lshr_b64 v[27:28], v[50:51], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[48:49], 16
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mul_f32_e32 v38, 1.0, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[29:30], v[38:39], 16
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_mul_f32_e32 v36, 1.0, v0
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[36:37], 16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_lshr_b64 v[31:32], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v0, v33
; SI-NEXT: v_mov_b32_e32 v1, v34
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -125707,10 +126448,18 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -125727,10 +126476,8 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; VI-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB77_5
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: s_branch .LBB77_6
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v17, s37
@@ -125763,7 +126510,7 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB77_5: ; %end
+; VI-NEXT: .LBB77_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -125816,10 +126563,18 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -125836,10 +126591,8 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; GFX9-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB77_5
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: s_branch .LBB77_6
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -125872,7 +126625,7 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB77_5: ; %end
+; GFX9-NEXT: .LBB77_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -125926,11 +126679,16 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB77_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -125948,8 +126706,6 @@ define inreg <64 x bfloat> @bitcast_v16f64_to_v64bf16_scalar(<16 x double> inreg
; GFX11-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB77_5
-; GFX11-NEXT: .LBB77_3:
-; GFX11-NEXT: s_branch .LBB77_2
; GFX11-NEXT: .LBB77_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -129497,7 +130253,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_mov_b32_e32 v33, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
@@ -129514,7 +130272,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v40, v56
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: s_cbranch_vccnz .LBB79_5
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -129884,10 +130642,18 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB79_3
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB79_4
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s50, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -130480,10 +131246,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB79_5
-; VI-NEXT: .LBB79_3:
-; VI-NEXT: s_branch .LBB79_2
-; VI-NEXT: .LBB79_4:
+; VI-NEXT: s_branch .LBB79_6
+; VI-NEXT: .LBB79_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -130516,7 +131280,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB79_5: ; %end
+; VI-NEXT: .LBB79_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v35, 15
; VI-NEXT: v_readlane_b32 s66, v35, 14
; VI-NEXT: v_readlane_b32 s65, v35, 13
@@ -130595,10 +131359,18 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB79_4
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s51, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -131209,10 +131981,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: s_branch .LBB79_5
-; GFX9-NEXT: .LBB79_3:
-; GFX9-NEXT: s_branch .LBB79_2
-; GFX9-NEXT: .LBB79_4:
+; GFX9-NEXT: s_branch .LBB79_6
+; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -131245,7 +132015,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB79_5: ; %end
+; GFX9-NEXT: .LBB79_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v36, 15
; GFX9-NEXT: v_readlane_b32 s66, v36, 14
; GFX9-NEXT: v_readlane_b32 s65, v36, 13
@@ -131325,11 +132095,16 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s37, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-TRUE16-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB79_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s51, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s51, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -131952,8 +132727,6 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v34
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v33.l
; GFX11-TRUE16-NEXT: s_branch .LBB79_5
-; GFX11-TRUE16-NEXT: .LBB79_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB79_2
; GFX11-TRUE16-NEXT: .LBB79_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -132051,11 +132824,16 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX11-FAKE16-NEXT: s_mov_b32 s37, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-FAKE16-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB79_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s51, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s51, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -132709,8 +133487,6 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v32, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v36, 16, v33
; GFX11-FAKE16-NEXT: s_branch .LBB79_5
-; GFX11-FAKE16-NEXT: .LBB79_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB79_2
; GFX11-FAKE16-NEXT: .LBB79_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -133231,7 +134007,7 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s69, s45, 16
; SI-NEXT: s_lshr_b32 s68, s43, 16
@@ -133262,11 +134038,51 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[28:29], s[42:43], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[30:31], s[44:45], 1.0
@@ -133319,42 +134135,8 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v31
-; SI-NEXT: s_branch .LBB81_5
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: s_branch .LBB81_6
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v32, s68
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -133406,9 +134188,9 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v42, s48
; SI-NEXT: v_mov_b32_e32 v43, s39
; SI-NEXT: v_mov_b32_e32 v44, s38
-; SI-NEXT: v_mov_b32_e32 v40, s36
-; SI-NEXT: v_mov_b32_e32 v39, s34
-; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v40, s34
+; SI-NEXT: v_mov_b32_e32 v39, s30
+; SI-NEXT: v_mov_b32_e32 v54, vcc_lo
; SI-NEXT: v_mov_b32_e32 v53, s94
; SI-NEXT: v_mov_b32_e32 v52, s92
; SI-NEXT: v_mov_b32_e32 v51, s90
@@ -133423,7 +134205,7 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v33, s60
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v32, s46
-; SI-NEXT: .LBB81_5: ; %end
+; SI-NEXT: .LBB81_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v39
@@ -133603,10 +134385,18 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -133623,10 +134413,8 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; VI-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB81_5
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: s_branch .LBB81_6
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v17, s37
@@ -133659,7 +134447,7 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB81_5: ; %end
+; VI-NEXT: .LBB81_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -133712,10 +134500,18 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -133732,10 +134528,8 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; GFX9-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB81_5
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: s_branch .LBB81_6
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -133768,7 +134562,7 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB81_5: ; %end
+; GFX9-NEXT: .LBB81_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -133822,11 +134616,16 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -133844,8 +134643,6 @@ define inreg <64 x half> @bitcast_v16f64_to_v64f16_scalar(<16 x double> inreg %a
; GFX11-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB81_5
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -134899,10 +135696,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_writelane_b32 v34, s30, 34
; SI-NEXT: v_writelane_b32 v34, s31, 35
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 vcc_hi, s8, 16
; SI-NEXT: v_readfirstlane_b32 s10, v15
; SI-NEXT: v_readfirstlane_b32 s12, v14
; SI-NEXT: v_readfirstlane_b32 s14, v13
@@ -134914,29 +135708,29 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_readfirstlane_b32 s90, v7
; SI-NEXT: v_readfirstlane_b32 s92, v6
; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s34, v3
-; SI-NEXT: v_readfirstlane_b32 s68, v2
-; SI-NEXT: v_readfirstlane_b32 s80, v1
-; SI-NEXT: v_readfirstlane_b32 s83, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v35, vcc_lo, 0
-; SI-NEXT: s_lshr_b32 s69, s29, 16
-; SI-NEXT: s_lshr_b32 s71, s28, 16
-; SI-NEXT: s_lshr_b32 s82, s27, 16
-; SI-NEXT: s_lshr_b32 s85, s26, 16
-; SI-NEXT: s_lshr_b32 s86, s25, 16
-; SI-NEXT: s_lshr_b32 s87, s24, 16
-; SI-NEXT: s_lshr_b32 s96, s23, 16
-; SI-NEXT: s_lshr_b32 s97, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s7, s19, 16
-; SI-NEXT: s_lshr_b32 s9, s18, 16
-; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: s_lshr_b32 s13, s16, 16
-; SI-NEXT: s_lshr_b32 s65, s10, 16
-; SI-NEXT: s_lshr_b32 s64, s12, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s30, v3
+; SI-NEXT: v_readfirstlane_b32 s34, v2
+; SI-NEXT: v_readfirstlane_b32 s70, v1
+; SI-NEXT: v_readfirstlane_b32 s81, v0
+; SI-NEXT: s_lshr_b32 s35, s29, 16
+; SI-NEXT: s_lshr_b32 s69, s28, 16
+; SI-NEXT: s_lshr_b32 s80, s27, 16
+; SI-NEXT: s_lshr_b32 s83, s26, 16
+; SI-NEXT: s_lshr_b32 s84, s25, 16
+; SI-NEXT: s_lshr_b32 s85, s24, 16
+; SI-NEXT: s_lshr_b32 s86, s23, 16
+; SI-NEXT: s_lshr_b32 s87, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s7, s17, 16
+; SI-NEXT: s_lshr_b32 s9, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s6, 16
+; SI-NEXT: s_lshr_b32 s66, s8, 16
+; SI-NEXT: s_lshr_b32 s11, s10, 16
+; SI-NEXT: s_lshr_b32 s13, s12, 16
; SI-NEXT: s_lshr_b32 s15, s14, 16
; SI-NEXT: s_lshr_b32 s73, s72, 16
; SI-NEXT: s_lshr_b32 s75, s74, 16
@@ -134946,74 +135740,75 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s31, s30, 16
-; SI-NEXT: s_lshr_b32 s35, s34, 16
-; SI-NEXT: s_lshr_b32 s70, s68, 16
-; SI-NEXT: s_lshr_b32 s81, s80, 16
-; SI-NEXT: s_lshr_b32 s84, s83, 16
+; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 s71, s70, 16
+; SI-NEXT: s_lshr_b32 s82, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v35, vcc_hi, 1
+; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v35, s65, 2
-; SI-NEXT: v_writelane_b32 v35, s64, 3
-; SI-NEXT: s_cbranch_scc0 .LBB83_3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v35, s67, 0
+; SI-NEXT: v_writelane_b32 v35, s66, 1
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s13, 16
+; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s9, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s11, 16
+; SI-NEXT: s_lshl_b32 s41, s7, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
-; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s30, 0xffff
; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_or_b32 s53, s4, s5
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -135043,22 +135838,31 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: s_lshl_b32 s5, s15, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB83_4
-; SI-NEXT: .LBB83_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s7
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -135070,7 +135874,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s9
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s99
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -135078,11 +135882,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s7
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s98
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s99
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s97
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -135093,7 +135897,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s96
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -135102,11 +135906,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s86
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -135117,7 +135921,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s85
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -135126,11 +135930,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s84
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s83
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -135141,7 +135945,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s80
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -135150,11 +135954,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: v_cvt_f32_f16_e32 v13, s28
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s35
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
@@ -135165,8 +135969,8 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v15, s29
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s84
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s81
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -135174,11 +135978,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s71
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s80
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s68
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
@@ -135186,11 +135990,11 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v18, s34
; SI-NEXT: v_or_b32_e32 v15, v16, v15
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
@@ -135198,10 +136002,10 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v19, 0x38000000, v19
; SI-NEXT: v_cvt_f16_f32_e32 v19, v19
; SI-NEXT: v_or_b32_e32 v16, v18, v16
-; SI-NEXT: v_cvt_f32_f16_e32 v18, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v18, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v19, v17
-; SI-NEXT: v_cvt_f32_f16_e32 v19, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v19, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v20, s95
; SI-NEXT: v_add_f32_e32 v18, 0x38000000, v18
; SI-NEXT: v_cvt_f16_f32_e32 v18, v18
@@ -135272,10 +136076,9 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v25, v27, v25
; SI-NEXT: v_cvt_f32_f16_e32 v27, s15
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_readlane_b32 s4, v35, 3
; SI-NEXT: v_or_b32_e32 v26, v28, v26
; SI-NEXT: v_cvt_f32_f16_e32 v28, s14
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s13
; SI-NEXT: v_add_f32_e32 v27, 0x38000000, v27
; SI-NEXT: v_cvt_f16_f32_e32 v27, v27
; SI-NEXT: v_add_f32_e32 v28, 0x38000000, v28
@@ -135283,11 +136086,10 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v28, v28
; SI-NEXT: v_cvt_f16_f32_e32 v29, v29
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; SI-NEXT: v_readlane_b32 s4, v35, 2
; SI-NEXT: v_cvt_f32_f16_e32 v30, s12
; SI-NEXT: v_or_b32_e32 v27, v28, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
-; SI-NEXT: v_cvt_f32_f16_e32 v29, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v29, s11
; SI-NEXT: v_cvt_f32_f16_e32 v31, s10
; SI-NEXT: v_add_f32_e32 v30, 0x38000000, v30
; SI-NEXT: v_cvt_f16_f32_e32 v30, v30
@@ -135316,11 +136118,8 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v30, v31, v30
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v33, v31
-; SI-NEXT: s_branch .LBB83_5
-; SI-NEXT: .LBB83_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB83_2
-; SI-NEXT: .LBB83_4:
+; SI-NEXT: s_branch .LBB83_6
+; SI-NEXT: .LBB83_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -135353,7 +136152,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB83_5: ; %end
+; SI-NEXT: .LBB83_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v34, 34
; SI-NEXT: v_readlane_b32 s31, v34, 35
; SI-NEXT: v_readlane_b32 s99, v34, 33
@@ -135453,10 +136252,18 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB83_3
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_4
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s51, 16
; VI-NEXT: v_mov_b32_e32 v16, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s4
@@ -135618,10 +136425,8 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s52, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB83_5
-; VI-NEXT: .LBB83_3:
-; VI-NEXT: s_branch .LBB83_2
-; VI-NEXT: .LBB83_4:
+; VI-NEXT: s_branch .LBB83_6
+; VI-NEXT: .LBB83_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -135654,7 +136459,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB83_5: ; %end
+; VI-NEXT: .LBB83_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v33, 15
; VI-NEXT: v_readlane_b32 s66, v33, 14
; VI-NEXT: v_readlane_b32 s65, v33, 13
@@ -135733,10 +136538,18 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_4
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s51, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s50, v16 op_sel_hi:[1,0]
@@ -135770,10 +136583,8 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v18, s54, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB83_5
-; GFX9-NEXT: .LBB83_3:
-; GFX9-NEXT: s_branch .LBB83_2
-; GFX9-NEXT: .LBB83_4:
+; GFX9-NEXT: s_branch .LBB83_6
+; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -135806,7 +136617,7 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB83_5: ; %end
+; GFX9-NEXT: .LBB83_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -135886,11 +136697,16 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB83_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s51 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s50 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s49 op_sel_hi:[0,1]
@@ -135924,8 +136740,6 @@ define inreg <16 x double> @bitcast_v64f16_to_v16f64_scalar(<64 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s53 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s52 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB83_5
-; GFX11-NEXT: .LBB83_3:
-; GFX11-NEXT: s_branch .LBB83_2
; GFX11-NEXT: .LBB83_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -136446,7 +137260,7 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB85_3
+; SI-NEXT: s_cbranch_scc0 .LBB85_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s69, s45, 16
; SI-NEXT: s_lshr_b32 s68, s43, 16
@@ -136477,11 +137291,51 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[20:21], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB85_4
-; SI-NEXT: .LBB85_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[20:21], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[18:19], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB85_3
+; SI-NEXT: .LBB85_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB85_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB85_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[28:29], s[42:43], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[30:31], s[44:45], 1.0
@@ -136534,42 +137388,8 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v31
-; SI-NEXT: s_branch .LBB85_5
-; SI-NEXT: .LBB85_3:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB85_2
-; SI-NEXT: .LBB85_4:
+; SI-NEXT: s_branch .LBB85_6
+; SI-NEXT: .LBB85_5:
; SI-NEXT: v_mov_b32_e32 v32, s68
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -136621,9 +137441,9 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v42, s48
; SI-NEXT: v_mov_b32_e32 v43, s39
; SI-NEXT: v_mov_b32_e32 v44, s38
-; SI-NEXT: v_mov_b32_e32 v40, s36
-; SI-NEXT: v_mov_b32_e32 v39, s34
-; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v40, s34
+; SI-NEXT: v_mov_b32_e32 v39, s30
+; SI-NEXT: v_mov_b32_e32 v54, vcc_lo
; SI-NEXT: v_mov_b32_e32 v53, s94
; SI-NEXT: v_mov_b32_e32 v52, s92
; SI-NEXT: v_mov_b32_e32 v51, s90
@@ -136638,7 +137458,7 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v33, s60
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v32, s46
-; SI-NEXT: .LBB85_5: ; %end
+; SI-NEXT: .LBB85_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v39
@@ -136818,10 +137638,18 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB85_3
+; VI-NEXT: s_cbranch_scc0 .LBB85_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB85_4
-; VI-NEXT: .LBB85_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB85_3
+; VI-NEXT: .LBB85_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB85_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB85_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -136838,10 +137666,8 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; VI-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB85_5
-; VI-NEXT: .LBB85_3:
-; VI-NEXT: s_branch .LBB85_2
-; VI-NEXT: .LBB85_4:
+; VI-NEXT: s_branch .LBB85_6
+; VI-NEXT: .LBB85_5:
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v17, s37
@@ -136874,7 +137700,7 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB85_5: ; %end
+; VI-NEXT: .LBB85_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 8
; VI-NEXT: v_readlane_b32 s31, v32, 9
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -136927,10 +137753,18 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB85_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB85_4
-; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB85_3
+; GFX9-NEXT: .LBB85_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB85_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -136947,10 +137781,8 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[20:21], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB85_5
-; GFX9-NEXT: .LBB85_3:
-; GFX9-NEXT: s_branch .LBB85_2
-; GFX9-NEXT: .LBB85_4:
+; GFX9-NEXT: s_branch .LBB85_6
+; GFX9-NEXT: .LBB85_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -136983,7 +137815,7 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB85_5: ; %end
+; GFX9-NEXT: .LBB85_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -137037,11 +137869,16 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
-; GFX11-NEXT: .LBB85_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB85_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -137059,8 +137896,6 @@ define inreg <64 x i16> @bitcast_v16f64_to_v64i16_scalar(<16 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[18:19], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB85_5
-; GFX11-NEXT: .LBB85_3:
-; GFX11-NEXT: s_branch .LBB85_2
; GFX11-NEXT: .LBB85_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -137943,13 +138778,10 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s99, 33
; SI-NEXT: v_writelane_b32 v32, s30, 34
; SI-NEXT: v_writelane_b32 v32, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s9, v16
-; SI-NEXT: s_lshr_b32 s14, s9, 16
-; SI-NEXT: v_readfirstlane_b32 s13, v14
-; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: v_readfirstlane_b32 s7, v17
+; SI-NEXT: v_readfirstlane_b32 s9, v16
; SI-NEXT: v_readfirstlane_b32 s11, v15
-; SI-NEXT: s_lshr_b32 s72, s13, 16
+; SI-NEXT: v_readfirstlane_b32 s13, v14
; SI-NEXT: v_readfirstlane_b32 s15, v13
; SI-NEXT: v_readfirstlane_b32 s73, v12
; SI-NEXT: v_readfirstlane_b32 s75, v11
@@ -137958,30 +138790,30 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s89, v8
; SI-NEXT: v_readfirstlane_b32 s91, v7
; SI-NEXT: v_readfirstlane_b32 s94, v6
-; SI-NEXT: v_readfirstlane_b32 s31, v5
-; SI-NEXT: v_readfirstlane_b32 s68, v4
-; SI-NEXT: v_readfirstlane_b32 s80, v3
-; SI-NEXT: v_readfirstlane_b32 s83, v2
-; SI-NEXT: v_readfirstlane_b32 s86, v1
-; SI-NEXT: v_readfirstlane_b32 s97, v0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; SI-NEXT: v_readfirstlane_b32 s34, v4
+; SI-NEXT: v_readfirstlane_b32 s70, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s84, v1
+; SI-NEXT: v_readfirstlane_b32 s87, v0
; SI-NEXT: s_lshr_b32 s92, s29, 16
; SI-NEXT: s_lshr_b32 s95, s28, 16
-; SI-NEXT: s_lshr_b32 s34, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s71, s25, 16
-; SI-NEXT: s_lshr_b32 s81, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s87, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s21, 16
-; SI-NEXT: s_lshr_b32 s99, s20, 16
-; SI-NEXT: s_lshr_b32 s6, s19, 16
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: s_lshr_b32 s10, s17, 16
-; SI-NEXT: s_lshr_b32 s12, s16, 16
-; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
-; SI-NEXT: s_lshr_b32 vcc_hi, s11, 16
+; SI-NEXT: s_lshr_b32 s30, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s69, s25, 16
+; SI-NEXT: s_lshr_b32 s71, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s85, s22, 16
+; SI-NEXT: s_lshr_b32 s96, s21, 16
+; SI-NEXT: s_lshr_b32 s97, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s99, s18, 16
+; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s8, s16, 16
+; SI-NEXT: s_lshr_b32 s67, s7, 16
+; SI-NEXT: s_lshr_b32 s10, s9, 16
+; SI-NEXT: s_lshr_b32 s12, s11, 16
+; SI-NEXT: s_lshr_b32 s14, s13, 16
; SI-NEXT: s_lshr_b32 s63, s15, 16
; SI-NEXT: s_lshr_b32 s62, s73, 16
; SI-NEXT: s_lshr_b32 s61, s75, 16
@@ -137990,54 +138822,55 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s68, 16
-; SI-NEXT: s_lshr_b32 s70, s80, 16
-; SI-NEXT: s_lshr_b32 s82, s83, 16
-; SI-NEXT: s_lshr_b32 s85, s86, 16
-; SI-NEXT: s_lshr_b32 s96, s97, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s34, 16
+; SI-NEXT: s_lshr_b32 s68, s70, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v33, s72, 1
+; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v33, s63, 2
-; SI-NEXT: v_writelane_b32 v33, s62, 3
-; SI-NEXT: s_cbranch_scc0 .LBB87_4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v33, s14, 0
+; SI-NEXT: v_writelane_b32 v33, s63, 1
+; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s10, 16
+; SI-NEXT: s_lshl_b32 s41, s6, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -138045,23 +138878,23 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s54, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s55, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -138077,92 +138910,109 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_or_b32 s59, s4, s5
; SI-NEXT: s_and_b32 s4, s77, 0xffff
; SI-NEXT: s_lshl_b32 s5, s60, 16
-; SI-NEXT: s_mov_b32 s76, s60
+; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: s_or_b32 s60, s4, s5
; SI-NEXT: s_and_b32 s4, s75, 0xffff
; SI-NEXT: s_lshl_b32 s5, s61, 16
-; SI-NEXT: s_mov_b32 s74, s61
+; SI-NEXT: s_mov_b32 s76, s61
; SI-NEXT: s_or_b32 s61, s4, s5
; SI-NEXT: s_and_b32 s4, s73, 0xffff
; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_mov_b32 s72, s62
; SI-NEXT: s_or_b32 s62, s4, s5
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s66, s4, s5
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
+; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
; SI-NEXT: s_or_b32 s67, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB87_3
-; SI-NEXT: .LBB87_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB87_3
+; SI-NEXT: .LBB87_2:
+; SI-NEXT: s_mov_b32 s74, s60
+; SI-NEXT: s_mov_b32 s76, s61
+; SI-NEXT: s_mov_b32 s72, s62
+; SI-NEXT: s_mov_b32 s14, s12
+; SI-NEXT: s_mov_b32 s12, s10
+; SI-NEXT: s_mov_b32 s10, s67
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB87_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB87_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, 16
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_add_i32 s37, s5, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s8, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -138174,35 +139024,35 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s97, s97, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
+; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s34, s34, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s54, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s55, s4, 0x30000
@@ -138227,27 +139077,26 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s74, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s75, s75, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s73, s73, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s72, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s15, s15, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 2
+; SI-NEXT: v_readlane_b32 s5, v33, 1
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s15, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 1
+; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s63, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s13, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
@@ -138255,21 +139104,20 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_add_i32 s64, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_add_i32 s65, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s66, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s14, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s67, s4, 0x30000
-; SI-NEXT: .LBB87_3: ; %end
+; SI-NEXT: .LBB87_5: ; %end
; SI-NEXT: v_readlane_b32 s30, v32, 34
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
@@ -138344,13 +139192,6 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB87_4:
-; SI-NEXT: s_mov_b32 s76, s60
-; SI-NEXT: s_mov_b32 s74, s61
-; SI-NEXT: s_mov_b32 s72, vcc_hi
-; SI-NEXT: s_mov_b32 s14, vcc_lo
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB87_2
;
; VI-LABEL: bitcast_v64i16_to_v16f64_scalar:
; VI: ; %bb.0:
@@ -138375,10 +139216,18 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB87_4
+; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB87_3
-; VI-NEXT: .LBB87_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB87_3
+; VI-NEXT: .LBB87_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB87_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB87_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s46, 3
; VI-NEXT: s_and_b32 s4, s46, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -138539,7 +139388,7 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s45, s4, 0x30000
-; VI-NEXT: .LBB87_3: ; %end
+; VI-NEXT: .LBB87_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -138573,8 +139422,6 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s7
; VI-NEXT: v_mov_b32_e32 v31, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB87_4:
-; VI-NEXT: s_branch .LBB87_2
;
; GFX9-LABEL: bitcast_v64i16_to_v16f64_scalar:
; GFX9: ; %bb.0:
@@ -138632,10 +139479,18 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB87_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB87_4
-; GFX9-NEXT: .LBB87_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB87_3
+; GFX9-NEXT: .LBB87_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB87_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -138668,10 +139523,8 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB87_5
-; GFX9-NEXT: .LBB87_3:
-; GFX9-NEXT: s_branch .LBB87_2
-; GFX9-NEXT: .LBB87_4:
+; GFX9-NEXT: s_branch .LBB87_6
+; GFX9-NEXT: .LBB87_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -138704,7 +139557,7 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB87_5: ; %end
+; GFX9-NEXT: .LBB87_6: ; %end
; GFX9-NEXT: v_readlane_b32 s67, v32, 15
; GFX9-NEXT: v_readlane_b32 s66, v32, 14
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
@@ -138784,11 +139637,16 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB87_4
-; GFX11-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB87_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s50, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s49, 3 op_sel_hi:[1,0]
@@ -138822,8 +139680,6 @@ define inreg <16 x double> @bitcast_v64i16_to_v16f64_scalar(<64 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB87_5
-; GFX11-NEXT: .LBB87_3:
-; GFX11-NEXT: s_branch .LBB87_2
; GFX11-NEXT: .LBB87_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -145255,663 +146111,757 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; SI-NEXT: v_writelane_b32 v47, s25, 4
; SI-NEXT: v_writelane_b32 v47, s24, 5
; SI-NEXT: v_writelane_b32 v47, s23, 6
-; SI-NEXT: v_writelane_b32 v47, s22, 7
-; SI-NEXT: v_writelane_b32 v47, s21, 8
-; SI-NEXT: v_writelane_b32 v47, s20, 9
-; SI-NEXT: v_writelane_b32 v47, s19, 10
-; SI-NEXT: v_writelane_b32 v47, s18, 11
-; SI-NEXT: v_writelane_b32 v47, s17, 12
-; SI-NEXT: v_writelane_b32 v47, s4, 13
+; SI-NEXT: v_writelane_b32 v47, s4, 7
; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_writelane_b32 v47, s4, 14
+; SI-NEXT: v_writelane_b32 v47, s4, 8
; SI-NEXT: v_readfirstlane_b32 s4, v28
-; SI-NEXT: v_writelane_b32 v47, s4, 15
+; SI-NEXT: v_writelane_b32 v47, s4, 9
; SI-NEXT: v_readfirstlane_b32 s4, v27
-; SI-NEXT: v_writelane_b32 v47, s4, 16
+; SI-NEXT: v_writelane_b32 v47, s4, 10
; SI-NEXT: v_readfirstlane_b32 s4, v26
-; SI-NEXT: v_writelane_b32 v47, s4, 17
+; SI-NEXT: v_writelane_b32 v47, s4, 11
; SI-NEXT: v_readfirstlane_b32 s4, v25
-; SI-NEXT: v_writelane_b32 v47, s4, 18
+; SI-NEXT: v_writelane_b32 v47, s4, 12
; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: v_writelane_b32 v47, s4, 19
+; SI-NEXT: v_writelane_b32 v47, s4, 13
; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: v_writelane_b32 v47, s4, 20
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:332
+; SI-NEXT: v_writelane_b32 v47, s4, 14
; SI-NEXT: v_readfirstlane_b32 s4, v22
-; SI-NEXT: v_writelane_b32 v47, s4, 21
+; SI-NEXT: v_writelane_b32 v47, s4, 15
; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: v_writelane_b32 v47, s4, 22
-; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:332
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:324
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:328
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:320
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:316
-; SI-NEXT: v_writelane_b32 v47, s4, 23
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:316
+; SI-NEXT: v_writelane_b32 v47, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:324
+; SI-NEXT: v_writelane_b32 v47, s4, 17
; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:328
-; SI-NEXT: v_writelane_b32 v47, s4, 24
+; SI-NEXT: v_writelane_b32 v47, s4, 18
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v47, s4, 25
+; SI-NEXT: v_writelane_b32 v47, s4, 19
; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_writelane_b32 v47, s4, 26
+; SI-NEXT: v_writelane_b32 v47, s4, 20
; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v47, s4, 27
+; SI-NEXT: v_writelane_b32 v47, s4, 21
; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: v_writelane_b32 v47, s4, 28
+; SI-NEXT: v_writelane_b32 v47, s4, 22
; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: s_mov_b32 s7, s16
-; SI-NEXT: v_writelane_b32 v47, s4, 29
-; SI-NEXT: v_readfirstlane_b32 s18, v13
-; SI-NEXT: v_readfirstlane_b32 s16, v12
+; SI-NEXT: v_writelane_b32 v47, s4, 23
+; SI-NEXT: v_readfirstlane_b32 s4, v13
+; SI-NEXT: v_writelane_b32 v47, s4, 24
+; SI-NEXT: v_readfirstlane_b32 s4, v12
+; SI-NEXT: v_writelane_b32 v47, s4, 25
+; SI-NEXT: s_mov_b32 s7, s17
+; SI-NEXT: v_readfirstlane_b32 s17, v10
+; SI-NEXT: v_readfirstlane_b32 s9, v9
; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:244
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:244
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
-; SI-NEXT: v_readfirstlane_b32 s23, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288
-; SI-NEXT: v_readfirstlane_b32 s19, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284
-; SI-NEXT: v_readfirstlane_b32 s25, v6
+; SI-NEXT: v_readfirstlane_b32 s55, v11
+; SI-NEXT: v_readfirstlane_b32 s15, v8
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:284
+; SI-NEXT: v_readfirstlane_b32 s67, v6
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:280
-; SI-NEXT: v_readfirstlane_b32 s11, v9
-; SI-NEXT: v_readfirstlane_b32 s48, v5
+; SI-NEXT: v_readfirstlane_b32 s51, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:276
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:264
-; SI-NEXT: v_readfirstlane_b32 s50, v4
+; SI-NEXT: v_readfirstlane_b32 s23, v4
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
-; SI-NEXT: v_readfirstlane_b32 s22, v3
+; SI-NEXT: v_readfirstlane_b32 s26, v3
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:308
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:304
+; SI-NEXT: v_readfirstlane_b32 s65, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:264
+; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:312
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:308
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:304
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300
-; SI-NEXT: v_readfirstlane_b32 s14, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:296
-; SI-NEXT: v_readfirstlane_b32 s24, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:292
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292
+; SI-NEXT: s_mov_b32 s6, s19
; SI-NEXT: ; implicit-def: $vgpr46 : SGPR spill to VGPR lane
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v20
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v23
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:256
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v1
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:248
-; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
-; SI-NEXT: v_readfirstlane_b32 s75, v22
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248
+; SI-NEXT: v_readfirstlane_b32 s4, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:232
+; SI-NEXT: v_readfirstlane_b32 s50, v22
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:216
-; SI-NEXT: v_readfirstlane_b32 s29, v24
+; SI-NEXT: v_readfirstlane_b32 s31, v25
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:208
-; SI-NEXT: v_readfirstlane_b32 s4, v21
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:208
+; SI-NEXT: v_readfirstlane_b32 s49, v20
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:224
-; SI-NEXT: v_readfirstlane_b32 s92, v19
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:236
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:232
-; SI-NEXT: v_writelane_b32 v47, s4, 30
-; SI-NEXT: v_readfirstlane_b32 s53, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:68
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:224
+; SI-NEXT: v_readfirstlane_b32 s80, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s28, v0
+; SI-NEXT: v_readfirstlane_b32 s81, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
-; SI-NEXT: v_readfirstlane_b32 s61, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
-; SI-NEXT: v_readfirstlane_b32 s41, v7
-; SI-NEXT: v_readfirstlane_b32 s88, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
-; SI-NEXT: v_readfirstlane_b32 s20, v5
-; SI-NEXT: v_readfirstlane_b32 s95, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s5, v11
+; SI-NEXT: v_readfirstlane_b32 s8, v8
+; SI-NEXT: v_writelane_b32 v47, s8, 26
+; SI-NEXT: v_readfirstlane_b32 s8, v6
+; SI-NEXT: v_writelane_b32 v47, s8, 27
+; SI-NEXT: v_readfirstlane_b32 s8, v5
+; SI-NEXT: v_writelane_b32 v47, s8, 28
+; SI-NEXT: v_readfirstlane_b32 s8, v4
+; SI-NEXT: v_writelane_b32 v47, s8, 29
+; SI-NEXT: v_readfirstlane_b32 s8, v3
+; SI-NEXT: v_writelane_b32 v47, s8, 30
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
-; SI-NEXT: v_readfirstlane_b32 s31, v4
+; SI-NEXT: v_readfirstlane_b32 s78, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
-; SI-NEXT: v_readfirstlane_b32 s76, v3
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:128
-; SI-NEXT: v_readfirstlane_b32 s34, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:156
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:148
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
+; SI-NEXT: v_readfirstlane_b32 s68, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:172
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
+; SI-NEXT: v_readfirstlane_b32 s35, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
+; SI-NEXT: v_readfirstlane_b32 s93, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
-; SI-NEXT: v_readfirstlane_b32 s40, v15
+; SI-NEXT: v_readfirstlane_b32 s53, v15
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:180
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
-; SI-NEXT: v_readfirstlane_b32 s59, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:172
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
-; SI-NEXT: v_readfirstlane_b32 s79, v18
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:204
+; SI-NEXT: v_readfirstlane_b32 s19, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
-; SI-NEXT: v_readfirstlane_b32 s93, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:196
+; SI-NEXT: v_readfirstlane_b32 s14, v18
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:196
; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
-; SI-NEXT: v_readfirstlane_b32 s89, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:164
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s65, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92
+; SI-NEXT: v_writelane_b32 v47, s8, 31
+; SI-NEXT: v_writelane_b32 v47, s18, 32
+; SI-NEXT: v_writelane_b32 v47, s6, 33
+; SI-NEXT: v_writelane_b32 v47, s7, 34
+; SI-NEXT: v_writelane_b32 v47, s16, 35
+; SI-NEXT: v_writelane_b32 v47, s22, 36
+; SI-NEXT: v_writelane_b32 v47, s21, 37
+; SI-NEXT: v_writelane_b32 v47, s20, 38
+; SI-NEXT: v_writelane_b32 v47, s9, 39
+; SI-NEXT: v_writelane_b32 v47, s15, 40
+; SI-NEXT: v_writelane_b32 v47, s51, 41
+; SI-NEXT: v_writelane_b32 v47, s23, 42
+; SI-NEXT: v_writelane_b32 v47, s55, 43
+; SI-NEXT: v_readfirstlane_b32 s94, v9
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
-; SI-NEXT: v_readfirstlane_b32 s27, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s30, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
-; SI-NEXT: v_readfirstlane_b32 s68, v1
+; SI-NEXT: v_readfirstlane_b32 s89, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
+; SI-NEXT: v_readfirstlane_b32 s54, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
+; SI-NEXT: v_readfirstlane_b32 s64, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
-; SI-NEXT: v_readfirstlane_b32 s47, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
-; SI-NEXT: v_readfirstlane_b32 s54, v22
+; SI-NEXT: v_readfirstlane_b32 s37, v22
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8
-; SI-NEXT: v_readfirstlane_b32 s67, v29
+; SI-NEXT: v_readfirstlane_b32 s79, v29
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32
-; SI-NEXT: v_readfirstlane_b32 s37, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:28
+; SI-NEXT: v_readfirstlane_b32 s57, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
-; SI-NEXT: v_readfirstlane_b32 s91, v25
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:20
+; SI-NEXT: v_readfirstlane_b32 s59, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16
-; SI-NEXT: v_readfirstlane_b32 s17, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
-; SI-NEXT: v_readfirstlane_b32 s64, v23
+; SI-NEXT: v_readfirstlane_b32 s38, v23
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
-; SI-NEXT: v_readfirstlane_b32 s94, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
-; SI-NEXT: v_readfirstlane_b32 s13, v14
-; SI-NEXT: v_writelane_b32 v46, s13, 0
-; SI-NEXT: v_readfirstlane_b32 s15, v52
-; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: v_readfirstlane_b32 s72, v33
-; SI-NEXT: v_readfirstlane_b32 s99, v53
-; SI-NEXT: v_readfirstlane_b32 s4, v35
-; SI-NEXT: v_writelane_b32 v47, s4, 31
-; SI-NEXT: v_readfirstlane_b32 s39, v24
-; SI-NEXT: v_readfirstlane_b32 s38, v8
-; SI-NEXT: v_readfirstlane_b32 s58, v39
-; SI-NEXT: v_readfirstlane_b32 s4, v4
-; SI-NEXT: v_writelane_b32 v47, s4, 32
-; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: v_writelane_b32 v47, s4, 33
-; SI-NEXT: v_writelane_b32 v47, s7, 34
-; SI-NEXT: v_readfirstlane_b32 s60, v3
-; SI-NEXT: v_writelane_b32 v47, s18, 35
+; SI-NEXT: v_writelane_b32 v47, s65, 44
+; SI-NEXT: v_writelane_b32 v47, s17, 45
+; SI-NEXT: v_writelane_b32 v47, s67, 46
+; SI-NEXT: v_writelane_b32 v47, s26, 47
+; SI-NEXT: v_writelane_b32 v47, vcc_hi, 48
+; SI-NEXT: v_writelane_b32 v47, s69, 49
+; SI-NEXT: v_writelane_b32 v47, s70, 50
+; SI-NEXT: v_readfirstlane_b32 s66, v25
+; SI-NEXT: v_readfirstlane_b32 s13, v12
+; SI-NEXT: v_readfirstlane_b32 s40, v52
+; SI-NEXT: v_readfirstlane_b32 s42, v0
+; SI-NEXT: v_readfirstlane_b32 s28, v53
+; SI-NEXT: v_readfirstlane_b32 s88, v7
+; SI-NEXT: v_readfirstlane_b32 s46, v39
+; SI-NEXT: v_readfirstlane_b32 s60, v2
+; SI-NEXT: v_readfirstlane_b32 s58, v48
+; SI-NEXT: v_readfirstlane_b32 s62, v3
; SI-NEXT: v_readfirstlane_b32 s63, v37
-; SI-NEXT: v_writelane_b32 v47, s16, 36
; SI-NEXT: v_readfirstlane_b32 s73, v38
-; SI-NEXT: v_writelane_b32 v47, s11, 37
+; SI-NEXT: v_readfirstlane_b32 s74, v5
+; SI-NEXT: v_readfirstlane_b32 s76, v35
+; SI-NEXT: v_readfirstlane_b32 s90, v36
+; SI-NEXT: v_readfirstlane_b32 s91, v4
+; SI-NEXT: v_readfirstlane_b32 s95, v8
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v33
+; SI-NEXT: v_readfirstlane_b32 s27, v34
+; SI-NEXT: v_readfirstlane_b32 s34, v6
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: v_writelane_b32 v47, s23, 38
-; SI-NEXT: v_readfirstlane_b32 s78, v36
-; SI-NEXT: v_writelane_b32 v47, s48, 39
-; SI-NEXT: v_writelane_b32 v47, s50, 40
-; SI-NEXT: v_readfirstlane_b32 s30, v15
-; SI-NEXT: v_writelane_b32 v47, s14, 41
-; SI-NEXT: v_readfirstlane_b32 s26, v28
-; SI-NEXT: v_writelane_b32 v47, s19, 42
-; SI-NEXT: v_writelane_b32 v47, s24, 43
-; SI-NEXT: v_writelane_b32 v47, s25, 44
-; SI-NEXT: v_writelane_b32 v47, s22, 45
-; SI-NEXT: v_readfirstlane_b32 s49, v30
-; SI-NEXT: v_writelane_b32 v47, vcc_lo, 46
-; SI-NEXT: v_writelane_b32 v47, vcc_hi, 47
-; SI-NEXT: v_writelane_b32 v47, s70, 48
-; SI-NEXT: v_readfirstlane_b32 s21, v11
-; SI-NEXT: v_readfirstlane_b32 s51, v17
-; SI-NEXT: v_readfirstlane_b32 s66, v26
-; SI-NEXT: v_readfirstlane_b32 s52, v27
-; SI-NEXT: v_readfirstlane_b32 s69, v16
-; SI-NEXT: v_readfirstlane_b32 s55, v18
-; SI-NEXT: v_readfirstlane_b32 s77, v31
-; SI-NEXT: v_readfirstlane_b32 s74, v10
-; SI-NEXT: v_readfirstlane_b32 s46, v32
-; SI-NEXT: v_readfirstlane_b32 s36, v7
-; SI-NEXT: v_readfirstlane_b32 s90, v34
-; SI-NEXT: v_readfirstlane_b32 s35, v6
-; SI-NEXT: v_readfirstlane_b32 s56, v12
-; SI-NEXT: v_readfirstlane_b32 s44, v49
-; SI-NEXT: v_readfirstlane_b32 s57, v13
-; SI-NEXT: v_readfirstlane_b32 s9, v51
-; SI-NEXT: v_writelane_b32 v46, s9, 1
-; SI-NEXT: v_writelane_b32 v46, s15, 2
-; SI-NEXT: v_readfirstlane_b32 s42, v1
-; SI-NEXT: v_writelane_b32 v46, s12, 3
-; SI-NEXT: v_writelane_b32 v46, s42, 4
-; SI-NEXT: v_readfirstlane_b32 s43, v50
-; SI-NEXT: v_writelane_b32 v46, s44, 5
-; SI-NEXT: v_writelane_b32 v46, s43, 6
-; SI-NEXT: v_readfirstlane_b32 s45, v2
-; SI-NEXT: v_writelane_b32 v46, s57, 7
-; SI-NEXT: v_writelane_b32 v46, s45, 8
-; SI-NEXT: v_readfirstlane_b32 s62, v48
-; SI-NEXT: v_writelane_b32 v46, s58, 9
-; SI-NEXT: v_writelane_b32 v46, s62, 10
-; SI-NEXT: v_writelane_b32 v46, s56, 11
-; SI-NEXT: v_writelane_b32 v46, s60, 12
-; SI-NEXT: v_writelane_b32 v46, s63, 13
-; SI-NEXT: v_writelane_b32 v46, s73, 14
-; SI-NEXT: v_writelane_b32 v46, s6, 15
-; SI-NEXT: v_writelane_b32 v46, s78, 16
-; SI-NEXT: v_writelane_b32 v46, s72, 17
-; SI-NEXT: v_writelane_b32 v46, s30, 18
-; SI-NEXT: v_writelane_b32 v46, s26, 19
-; SI-NEXT: v_writelane_b32 v46, s27, 20
-; SI-NEXT: v_writelane_b32 v46, s53, 21
-; SI-NEXT: v_writelane_b32 v46, s65, 22
-; SI-NEXT: v_writelane_b32 v46, s49, 23
+; SI-NEXT: v_readfirstlane_b32 s72, v13
+; SI-NEXT: v_readfirstlane_b32 s75, v31
+; SI-NEXT: v_readfirstlane_b32 s45, v32
+; SI-NEXT: v_readfirstlane_b32 s25, v11
+; SI-NEXT: v_readfirstlane_b32 s24, v15
+; SI-NEXT: v_readfirstlane_b32 s77, v28
+; SI-NEXT: v_readfirstlane_b32 s12, v30
+; SI-NEXT: v_readfirstlane_b32 s36, v14
+; SI-NEXT: v_readfirstlane_b32 s52, v18
+; SI-NEXT: v_readfirstlane_b32 s92, v26
+; SI-NEXT: v_readfirstlane_b32 s39, v27
+; SI-NEXT: v_readfirstlane_b32 s48, v19
+; SI-NEXT: v_readfirstlane_b32 s44, v17
+; SI-NEXT: v_readfirstlane_b32 s61, v9
+; SI-NEXT: v_readfirstlane_b32 s47, v49
+; SI-NEXT: v_readfirstlane_b32 s56, v10
+; SI-NEXT: v_readfirstlane_b32 s10, v51
+; SI-NEXT: v_readfirstlane_b32 s8, v21
+; SI-NEXT: v_readfirstlane_b32 s11, v54
+; SI-NEXT: s_waitcnt vmcnt(13)
+; SI-NEXT: v_readfirstlane_b32 s29, v16
+; SI-NEXT: v_writelane_b32 v46, s11, 0
+; SI-NEXT: v_writelane_b32 v46, s29, 1
+; SI-NEXT: v_writelane_b32 v46, s13, 2
+; SI-NEXT: v_writelane_b32 v46, s10, 3
+; SI-NEXT: v_writelane_b32 v46, s40, 4
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s71, v29
-; SI-NEXT: v_writelane_b32 v46, s17, 24
-; SI-NEXT: v_readfirstlane_b32 s80, v43
-; SI-NEXT: v_writelane_b32 v47, s71, 49
-; SI-NEXT: v_writelane_b32 v46, s54, 25
+; SI-NEXT: v_readfirstlane_b32 s41, v1
+; SI-NEXT: v_writelane_b32 v46, s42, 5
+; SI-NEXT: v_writelane_b32 v46, s41, 6
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s81, v44
-; SI-NEXT: v_writelane_b32 v47, s80, 50
-; SI-NEXT: v_writelane_b32 v46, s21, 26
+; SI-NEXT: v_readfirstlane_b32 s43, v50
+; SI-NEXT: v_writelane_b32 v46, s47, 7
+; SI-NEXT: v_writelane_b32 v46, s43, 8
+; SI-NEXT: v_writelane_b32 v46, s56, 9
+; SI-NEXT: v_writelane_b32 v46, s60, 10
+; SI-NEXT: v_writelane_b32 v46, s46, 11
+; SI-NEXT: v_writelane_b32 v46, s58, 12
+; SI-NEXT: v_writelane_b32 v46, s61, 13
+; SI-NEXT: v_writelane_b32 v46, s62, 14
+; SI-NEXT: v_writelane_b32 v46, s63, 15
+; SI-NEXT: v_writelane_b32 v46, s73, 16
+; SI-NEXT: v_writelane_b32 v46, s88, 17
+; SI-NEXT: v_writelane_b32 v46, s74, 18
+; SI-NEXT: v_writelane_b32 v46, s76, 19
+; SI-NEXT: v_writelane_b32 v46, s90, 20
+; SI-NEXT: v_writelane_b32 v46, s91, 21
+; SI-NEXT: v_writelane_b32 v46, s95, 22
+; SI-NEXT: v_writelane_b32 v46, vcc_lo, 23
+; SI-NEXT: v_writelane_b32 v46, s27, 24
+; SI-NEXT: v_writelane_b32 v46, s34, 25
+; SI-NEXT: v_writelane_b32 v46, s5, 26
+; SI-NEXT: v_writelane_b32 v46, s72, 27
+; SI-NEXT: v_writelane_b32 v46, s75, 28
+; SI-NEXT: v_writelane_b32 v46, s45, 29
+; SI-NEXT: v_writelane_b32 v46, s78, 30
+; SI-NEXT: v_writelane_b32 v46, s25, 31
+; SI-NEXT: v_writelane_b32 v46, s24, 32
+; SI-NEXT: v_writelane_b32 v46, s77, 33
+; SI-NEXT: v_writelane_b32 v46, s30, 34
+; SI-NEXT: v_writelane_b32 v46, s80, 35
+; SI-NEXT: v_writelane_b32 v46, s94, 36
+; SI-NEXT: v_writelane_b32 v46, s12, 37
+; SI-NEXT: v_writelane_b32 v46, s54, 38
+; SI-NEXT: v_writelane_b32 v46, s37, 39
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s82, v25
-; SI-NEXT: v_writelane_b32 v47, s81, 51
-; SI-NEXT: v_writelane_b32 v46, s64, 27
-; SI-NEXT: v_readfirstlane_b32 s83, v41
-; SI-NEXT: v_writelane_b32 v47, s82, 52
-; SI-NEXT: v_writelane_b32 v46, s67, 28
+; SI-NEXT: v_readfirstlane_b32 s71, v29
+; SI-NEXT: v_writelane_b32 v46, s36, 40
+; SI-NEXT: v_readfirstlane_b32 s83, v43
+; SI-NEXT: v_writelane_b32 v47, s71, 51
+; SI-NEXT: v_writelane_b32 v46, s38, 41
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s84, v42
-; SI-NEXT: v_writelane_b32 v47, s83, 53
-; SI-NEXT: v_writelane_b32 v46, s51, 29
-; SI-NEXT: v_readfirstlane_b32 s85, v22
-; SI-NEXT: v_writelane_b32 v47, s84, 54
-; SI-NEXT: v_writelane_b32 v46, s66, 30
+; SI-NEXT: v_readfirstlane_b32 s82, v44
+; SI-NEXT: v_writelane_b32 v47, s83, 52
+; SI-NEXT: v_writelane_b32 v46, s79, 42
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s86, v23
+; SI-NEXT: v_readfirstlane_b32 s84, v24
+; SI-NEXT: v_writelane_b32 v47, s82, 53
+; SI-NEXT: v_writelane_b32 v46, s52, 43
+; SI-NEXT: v_readfirstlane_b32 s85, v41
+; SI-NEXT: v_writelane_b32 v47, s84, 54
+; SI-NEXT: v_writelane_b32 v46, s92, 44
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_readfirstlane_b32 s86, v42
; SI-NEXT: v_writelane_b32 v47, s85, 55
-; SI-NEXT: v_writelane_b32 v46, s68, 31
-; SI-NEXT: v_readfirstlane_b32 s87, v55
+; SI-NEXT: v_writelane_b32 v46, s64, 45
+; SI-NEXT: v_readfirstlane_b32 s87, v22
; SI-NEXT: v_writelane_b32 v47, s86, 56
-; SI-NEXT: v_writelane_b32 v46, s28, 32
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s96, v40
+; SI-NEXT: v_writelane_b32 v46, s81, 46
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s96, v23
; SI-NEXT: v_writelane_b32 v47, s87, 57
-; SI-NEXT: v_writelane_b32 v46, s52, 33
-; SI-NEXT: v_readfirstlane_b32 s97, v21
+; SI-NEXT: v_writelane_b32 v46, s39, 47
+; SI-NEXT: v_readfirstlane_b32 s97, v55
; SI-NEXT: v_writelane_b32 v47, s96, 58
-; SI-NEXT: v_writelane_b32 v46, s37, 34
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s98, v20
+; SI-NEXT: v_writelane_b32 v46, s57, 48
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s98, v40
; SI-NEXT: v_writelane_b32 v47, s97, 59
-; SI-NEXT: v_writelane_b32 v46, s69, 35
+; SI-NEXT: v_writelane_b32 v46, s48, 49
+; SI-NEXT: v_readfirstlane_b32 s99, v20
; SI-NEXT: v_writelane_b32 v47, s98, 60
-; SI-NEXT: v_writelane_b32 v46, s94, 36
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s10, v54
+; SI-NEXT: v_writelane_b32 v46, s89, 50
; SI-NEXT: v_writelane_b32 v47, s99, 61
-; SI-NEXT: v_writelane_b32 v46, s91, 37
-; SI-NEXT: v_readfirstlane_b32 s8, v19
-; SI-NEXT: v_writelane_b32 v47, s10, 62
-; SI-NEXT: v_writelane_b32 v46, s39, 38
-; SI-NEXT: v_writelane_b32 v47, s8, 63
-; SI-NEXT: v_writelane_b32 v46, s55, 39
-; SI-NEXT: s_cbranch_scc0 .LBB89_4
+; SI-NEXT: v_writelane_b32 v46, s59, 51
+; SI-NEXT: v_writelane_b32 v47, s8, 62
+; SI-NEXT: v_writelane_b32 v46, s66, 52
+; SI-NEXT: v_writelane_b32 v47, s28, 63
+; SI-NEXT: v_writelane_b32 v46, s44, 53
+; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_writelane_b32 v46, s38, 42
-; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: v_readlane_b32 s5, v47, 12
-; SI-NEXT: v_writelane_b32 v46, s20, 43
+; SI-NEXT: v_writelane_b32 v46, s4, 56
+; SI-NEXT: s_and_b32 s4, s16, 0xff
+; SI-NEXT: v_writelane_b32 v46, s49, 57
+; SI-NEXT: s_mov_b32 s49, s31
+; SI-NEXT: s_mov_b32 s31, s19
+; SI-NEXT: s_mov_b32 s19, s14
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s5, 24
-; SI-NEXT: v_writelane_b32 v46, s90, 44
-; SI-NEXT: v_writelane_b32 v46, s88, 45
-; SI-NEXT: s_or_b32 s88, s5, s4
-; SI-NEXT: v_readlane_b32 s4, v47, 11
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s5, v47, 10
+; SI-NEXT: s_mov_b32 s14, s5
+; SI-NEXT: s_lshl_b32 s5, s7, 24
+; SI-NEXT: s_or_b32 s16, s5, s4
+; SI-NEXT: s_and_b32 s4, s18, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s5, 24
-; SI-NEXT: s_or_b32 s38, s5, s4
-; SI-NEXT: v_readlane_b32 s4, v47, 9
-; SI-NEXT: v_readlane_b32 s5, v47, 8
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: v_writelane_b32 v46, s36, 46
+; SI-NEXT: s_lshl_b32 s5, s6, 24
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_writelane_b32 v46, s4, 54
+; SI-NEXT: s_and_b32 s4, s20, 0xff
+; SI-NEXT: s_lshl_b32 s5, s21, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 47
-; SI-NEXT: v_readlane_b32 s4, v47, 7
-; SI-NEXT: s_and_b32 s5, s4, 0xff
+; SI-NEXT: v_writelane_b32 v46, s4, 58
+; SI-NEXT: s_and_b32 s5, s22, 0xff
; SI-NEXT: v_readlane_b32 s4, v47, 6
-; SI-NEXT: s_mov_b32 s90, s6
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s4, 24
; SI-NEXT: v_readlane_b32 s4, v47, 5
-; SI-NEXT: s_or_b32 s20, s6, s5
+; SI-NEXT: s_or_b32 s18, s6, s5
; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 4
+; SI-NEXT: v_readlane_b32 s6, v47, 4
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s4, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s4, v47, 3
-; SI-NEXT: v_writelane_b32 v46, s5, 41
-; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 2
+; SI-NEXT: v_writelane_b32 v46, s5, 55
+; SI-NEXT: v_readlane_b32 s5, v47, 3
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: v_readlane_b32 s6, v47, 2
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s4, 24
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s4, v47, 1
-; SI-NEXT: v_writelane_b32 v46, s5, 40
-; SI-NEXT: s_and_b32 s5, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 0
-; SI-NEXT: s_lshl_b32 s6, s4, 8
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_or_b32 s20, s6, s5
+; SI-NEXT: v_readlane_b32 s5, v47, 1
+; SI-NEXT: v_readlane_b32 s6, v47, 0
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: s_or_b32 s4, s5, s6
; SI-NEXT: s_and_b32 s6, s70, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, vcc_lo, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 48
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 24
+; SI-NEXT: v_writelane_b32 v46, s4, 59
; SI-NEXT: s_or_b32 s4, s7, s6
-; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
+; SI-NEXT: s_and_b32 s6, s69, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s22, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 49
+; SI-NEXT: s_lshl_b32 s7, s26, 24
+; SI-NEXT: v_writelane_b32 v46, s4, 60
; SI-NEXT: s_or_b32 s4, s7, s6
-; SI-NEXT: s_and_b32 s6, s50, 0xff
+; SI-NEXT: s_and_b32 s6, s23, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s48, 24
-; SI-NEXT: s_or_b32 s22, s7, s6
-; SI-NEXT: s_and_b32 s6, s25, 0xff
-; SI-NEXT: s_lshl_b32 s7, s19, 8
-; SI-NEXT: s_or_b32 s70, s6, s7
-; SI-NEXT: s_and_b32 s7, s23, 0xff
+; SI-NEXT: s_lshl_b32 s7, s51, 24
+; SI-NEXT: s_or_b32 s21, s7, s6
+; SI-NEXT: s_and_b32 s6, s67, 0xff
+; SI-NEXT: s_lshl_b32 s7, s65, 8
+; SI-NEXT: s_or_b32 vcc_hi, s6, s7
+; SI-NEXT: s_and_b32 s7, s15, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s11, s11, 24
-; SI-NEXT: s_or_b32 s23, s11, s7
-; SI-NEXT: s_and_b32 s7, s24, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s11, s14, 24
-; SI-NEXT: v_writelane_b32 v46, s4, 50
-; SI-NEXT: s_or_b32 s4, s11, s7
-; SI-NEXT: s_and_b32 s7, s16, 0xff
-; SI-NEXT: v_writelane_b32 v46, s4, 51
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_or_b32 s22, s9, s7
+; SI-NEXT: s_and_b32 s7, s17, 0xff
+; SI-NEXT: v_writelane_b32 v46, s4, 61
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s11, s18, 24
-; SI-NEXT: v_readlane_b32 s4, v47, 29
-; SI-NEXT: s_or_b32 s6, s11, s7
-; SI-NEXT: s_and_b32 s7, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 28
-; SI-NEXT: s_lshl_b32 s11, s4, 8
-; SI-NEXT: v_readlane_b32 s4, v47, 27
-; SI-NEXT: s_or_b32 s7, s7, s11
-; SI-NEXT: s_and_b32 s11, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 26
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s14, s4, 24
+; SI-NEXT: s_lshl_b32 s9, s55, 24
; SI-NEXT: v_readlane_b32 s4, v47, 25
-; SI-NEXT: s_or_b32 s25, s14, s11
-; SI-NEXT: s_and_b32 s11, s4, 0xff
+; SI-NEXT: s_or_b32 s70, s9, s7
+; SI-NEXT: s_and_b32 s7, s4, 0xff
; SI-NEXT: v_readlane_b32 s4, v47, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s14, s4, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s4, 24
; SI-NEXT: v_readlane_b32 s4, v47, 23
-; SI-NEXT: s_or_b32 s24, s14, s11
-; SI-NEXT: s_and_b32 s11, s4, 0xff
+; SI-NEXT: s_or_b32 s23, s9, s7
+; SI-NEXT: s_and_b32 s7, s4, 0xff
; SI-NEXT: v_readlane_b32 s4, v47, 22
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s14, s4, 24
+; SI-NEXT: s_lshl_b32 s9, s4, 8
; SI-NEXT: v_readlane_b32 s4, v47, 21
-; SI-NEXT: s_or_b32 s48, s14, s11
-; SI-NEXT: s_and_b32 s11, s4, 0xff
+; SI-NEXT: s_or_b32 s5, s7, s9
+; SI-NEXT: s_and_b32 s7, s4, 0xff
; SI-NEXT: v_readlane_b32 s4, v47, 20
-; SI-NEXT: s_lshl_b32 s14, s4, 8
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s4, 24
; SI-NEXT: v_readlane_b32 s4, v47, 19
-; SI-NEXT: s_or_b32 s5, s11, s14
-; SI-NEXT: s_and_b32 s11, s4, 0xff
-; SI-NEXT: v_readlane_b32 s4, v47, 18
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s14, s4, 24
-; SI-NEXT: s_or_b32 s4, s14, s11
-; SI-NEXT: v_readlane_b32 s11, v47, 17
-; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: v_readlane_b32 s14, v47, 16
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s16, s14, 24
-; SI-NEXT: s_or_b32 s50, s16, s11
-; SI-NEXT: v_readlane_b32 s11, v47, 15
-; SI-NEXT: s_and_b32 s16, s11, 0xff
-; SI-NEXT: v_readlane_b32 s11, v47, 14
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s18, s11, 24
-; SI-NEXT: v_readlane_b32 s11, v47, 13
-; SI-NEXT: s_or_b32 s14, s18, s16
-; SI-NEXT: s_and_b32 s16, s11, 0xff
-; SI-NEXT: s_lshl_b32 s18, s81, 8
-; SI-NEXT: s_or_b32 s18, s16, s18
-; SI-NEXT: s_and_b32 s16, s71, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s80, 24
-; SI-NEXT: s_or_b32 s80, s19, s16
-; SI-NEXT: s_and_b32 s16, s85, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s84, 24
-; SI-NEXT: s_or_b32 s71, s19, s16
-; SI-NEXT: s_and_b32 s16, s82, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s83, 24
-; SI-NEXT: s_or_b32 s81, s19, s16
-; SI-NEXT: s_and_b32 s16, s97, 0xff
-; SI-NEXT: s_lshl_b32 s19, s96, 8
-; SI-NEXT: s_or_b32 vcc_lo, s16, s19
-; SI-NEXT: s_and_b32 s16, s86, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s87, 24
-; SI-NEXT: s_or_b32 s82, s19, s16
-; SI-NEXT: s_and_b32 s16, s8, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s10, 24
-; SI-NEXT: s_or_b32 s10, s19, s16
-; SI-NEXT: s_and_b32 s16, s98, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s99, 24
-; SI-NEXT: s_or_b32 s83, s19, s16
-; SI-NEXT: s_and_b32 s16, s12, 0xff
-; SI-NEXT: s_lshl_b32 s19, s15, 8
-; SI-NEXT: s_or_b32 vcc_hi, s16, s19
-; SI-NEXT: s_and_b32 s16, s13, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s9, 24
-; SI-NEXT: s_or_b32 s13, s19, s16
-; SI-NEXT: s_and_b32 s16, s57, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s43, 24
-; SI-NEXT: s_or_b32 s12, s19, s16
-; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s44, 24
-; SI-NEXT: s_or_b32 s96, s19, s16
-; SI-NEXT: s_and_b32 s16, s56, 0xff
-; SI-NEXT: s_lshl_b32 s19, s62, 8
-; SI-NEXT: s_or_b32 s84, s16, s19
-; SI-NEXT: s_and_b32 s16, s45, 0xff
-; SI-NEXT: v_readlane_b32 s8, v47, 33
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s58, 24
+; SI-NEXT: s_or_b32 s26, s9, s7
+; SI-NEXT: s_and_b32 s7, s4, 0xff
+; SI-NEXT: v_readlane_b32 s9, v47, 18
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_or_b32 s6, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 17
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s9, v47, 16
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_or_b32 s69, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 15
+; SI-NEXT: v_readlane_b32 s9, v47, 14
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_or_b32 s4, s7, s9
+; SI-NEXT: v_readlane_b32 s7, v47, 13
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s9, v47, 12
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_or_b32 s55, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 11
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s9, v47, 10
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_or_b32 s51, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 9
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s9, v47, 8
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s15, s9, 24
+; SI-NEXT: s_or_b32 s65, s15, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 7
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s17, s82, 8
+; SI-NEXT: s_or_b32 s82, s7, s17
+; SI-NEXT: s_and_b32 s7, s71, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s83, 24
+; SI-NEXT: s_or_b32 s71, s17, s7
+; SI-NEXT: s_and_b32 s7, s87, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s86, 24
+; SI-NEXT: s_or_b32 s67, s17, s7
+; SI-NEXT: s_and_b32 s7, s84, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s85, 24
+; SI-NEXT: s_or_b32 s86, s17, s7
+; SI-NEXT: s_and_b32 s7, s99, 0xff
+; SI-NEXT: s_lshl_b32 s17, s98, 8
+; SI-NEXT: s_or_b32 s83, s7, s17
+; SI-NEXT: s_and_b32 s7, s96, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s97, 24
+; SI-NEXT: s_or_b32 s87, s17, s7
+; SI-NEXT: s_and_b32 s7, s29, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s11, 24
+; SI-NEXT: s_or_b32 s11, s17, s7
+; SI-NEXT: s_and_b32 s7, s8, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s28, 24
+; SI-NEXT: s_or_b32 s9, s17, s7
+; SI-NEXT: s_and_b32 s7, s42, 0xff
+; SI-NEXT: s_lshl_b32 s17, s40, 8
+; SI-NEXT: s_or_b32 s84, s7, s17
+; SI-NEXT: s_and_b32 s7, s13, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s10, 24
+; SI-NEXT: s_or_b32 s15, s17, s7
+; SI-NEXT: s_and_b32 s7, s56, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s43, 24
+; SI-NEXT: s_or_b32 s29, s17, s7
+; SI-NEXT: s_and_b32 s7, s41, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s47, 24
+; SI-NEXT: s_or_b32 s40, s17, s7
+; SI-NEXT: s_and_b32 s7, s61, 0xff
+; SI-NEXT: s_lshl_b32 s17, s58, 8
+; SI-NEXT: s_or_b32 s85, s7, s17
+; SI-NEXT: s_and_b32 s7, s60, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s46, 24
+; SI-NEXT: s_or_b32 s42, s17, s7
+; SI-NEXT: s_and_b32 s7, s88, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s73, 24
+; SI-NEXT: s_or_b32 s41, s17, s7
+; SI-NEXT: s_and_b32 s7, s62, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s63, 24
+; SI-NEXT: s_or_b32 s28, s17, s7
+; SI-NEXT: s_and_b32 s7, s91, 0xff
+; SI-NEXT: s_lshl_b32 s17, s90, 8
+; SI-NEXT: s_mov_b32 s88, s86
+; SI-NEXT: s_or_b32 s86, s7, s17
+; SI-NEXT: s_and_b32 s7, s74, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s76, 24
+; SI-NEXT: s_or_b32 s47, s17, s7
+; SI-NEXT: s_and_b32 s7, s34, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s27, 24
+; SI-NEXT: s_or_b32 s43, s17, s7
+; SI-NEXT: s_and_b32 s7, s95, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, vcc_lo, 24
+; SI-NEXT: s_or_b32 s27, s17, s7
+; SI-NEXT: s_and_b32 s7, s25, 0xff
+; SI-NEXT: s_lshl_b32 s17, s45, 8
+; SI-NEXT: s_mov_b32 s61, s21
+; SI-NEXT: s_or_b32 s21, s7, s17
+; SI-NEXT: s_and_b32 s7, s72, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s75, 24
+; SI-NEXT: s_or_b32 s72, s17, s7
+; SI-NEXT: s_and_b32 s7, s36, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s12, 24
+; SI-NEXT: s_or_b32 s25, s17, s7
+; SI-NEXT: s_and_b32 s7, s24, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s77, 24
+; SI-NEXT: s_or_b32 s74, s17, s7
+; SI-NEXT: s_and_b32 s7, s48, 0xff
+; SI-NEXT: s_lshl_b32 s17, s39, 8
+; SI-NEXT: s_mov_b32 s46, s16
+; SI-NEXT: s_or_b32 s16, s7, s17
+; SI-NEXT: s_and_b32 s7, s52, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s92, 24
+; SI-NEXT: s_or_b32 s52, s17, s7
+; SI-NEXT: s_and_b32 s7, s44, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s66, 24
+; SI-NEXT: s_mov_b32 s75, s51
+; SI-NEXT: s_or_b32 s51, s17, s7
+; SI-NEXT: s_and_b32 s7, s79, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s37, 24
+; SI-NEXT: s_or_b32 s76, s17, s7
+; SI-NEXT: s_and_b32 s7, s59, 0xff
+; SI-NEXT: s_lshl_b32 s17, s57, 8
; SI-NEXT: s_mov_b32 s62, s22
-; SI-NEXT: s_or_b32 s22, s19, s16
-; SI-NEXT: s_and_b32 s16, s8, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s73, 24
-; SI-NEXT: s_or_b32 s8, s19, s16
-; SI-NEXT: s_and_b32 s16, s60, 0xff
-; SI-NEXT: v_readlane_b32 s9, v47, 32
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s63, 24
-; SI-NEXT: s_or_b32 s15, s19, s16
-; SI-NEXT: s_and_b32 s16, s9, 0xff
-; SI-NEXT: s_lshl_b32 s19, s78, 8
-; SI-NEXT: s_or_b32 s85, s16, s19
-; SI-NEXT: s_and_b32 s16, s90, 0xff
-; SI-NEXT: v_readlane_b32 s9, v47, 31
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s9, 24
-; SI-NEXT: s_mov_b32 s58, s24
-; SI-NEXT: v_readlane_b32 s90, v46, 44
-; SI-NEXT: s_or_b32 s24, s19, s16
-; SI-NEXT: s_and_b32 s16, s35, 0xff
-; SI-NEXT: v_readlane_b32 s36, v46, 46
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s90, 24
-; SI-NEXT: s_or_b32 s9, s19, s16
-; SI-NEXT: s_and_b32 s16, s36, 0xff
-; SI-NEXT: s_mov_b32 s57, s38
-; SI-NEXT: v_readlane_b32 s38, v46, 42
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s72, 24
-; SI-NEXT: s_or_b32 s11, s19, s16
-; SI-NEXT: s_and_b32 s16, s38, 0xff
-; SI-NEXT: s_lshl_b32 s19, s46, 8
-; SI-NEXT: s_or_b32 s86, s16, s19
-; SI-NEXT: s_and_b32 s16, s74, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s77, 24
-; SI-NEXT: s_or_b32 s43, s19, s16
-; SI-NEXT: s_and_b32 s16, s21, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s49, 24
-; SI-NEXT: s_or_b32 s44, s19, s16
-; SI-NEXT: s_and_b32 s16, s30, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s26, 24
-; SI-NEXT: s_or_b32 s42, s19, s16
-; SI-NEXT: s_and_b32 s16, s69, 0xff
-; SI-NEXT: s_lshl_b32 s19, s52, 8
-; SI-NEXT: s_or_b32 s87, s16, s19
-; SI-NEXT: s_and_b32 s16, s51, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s66, 24
-; SI-NEXT: s_or_b32 s45, s19, s16
-; SI-NEXT: s_and_b32 s16, s55, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s39, 24
-; SI-NEXT: s_mov_b32 s66, s15
-; SI-NEXT: s_or_b32 s15, s19, s16
-; SI-NEXT: s_and_b32 s16, s67, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s54, 24
-; SI-NEXT: s_or_b32 s30, s19, s16
-; SI-NEXT: s_and_b32 s16, s91, 0xff
-; SI-NEXT: s_lshl_b32 s19, s37, 8
-; SI-NEXT: s_mov_b32 s63, s23
-; SI-NEXT: s_or_b32 s23, s16, s19
-; SI-NEXT: s_and_b32 s16, s64, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s17, 24
-; SI-NEXT: s_or_b32 s67, s19, s16
-; SI-NEXT: s_and_b32 s16, s94, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s28, 24
-; SI-NEXT: s_mov_b32 s55, s13
-; SI-NEXT: s_or_b32 s13, s19, s16
-; SI-NEXT: s_and_b32 s16, s53, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s27, 24
-; SI-NEXT: s_or_b32 s21, s19, s16
-; SI-NEXT: s_and_b32 s16, s68, 0xff
-; SI-NEXT: s_lshl_b32 s19, s65, 8
-; SI-NEXT: s_or_b32 s26, s16, s19
-; SI-NEXT: s_and_b32 s16, s47, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s95, 24
-; SI-NEXT: s_mov_b32 s54, s12
-; SI-NEXT: s_or_b32 s12, s19, s16
-; SI-NEXT: s_and_b32 s16, s76, 0xff
+; SI-NEXT: s_or_b32 s22, s7, s17
+; SI-NEXT: s_and_b32 s7, s38, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s54, 24
+; SI-NEXT: s_mov_b32 s77, s55
+; SI-NEXT: s_or_b32 s55, s17, s7
+; SI-NEXT: s_and_b32 s7, s89, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s81, 24
+; SI-NEXT: s_or_b32 s54, s17, s7
+; SI-NEXT: s_and_b32 s7, s80, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s30, 24
+; SI-NEXT: s_or_b32 s45, s17, s7
+; SI-NEXT: s_and_b32 s7, s64, 0xff
+; SI-NEXT: s_lshl_b32 s17, s94, 8
; SI-NEXT: s_mov_b32 s60, s20
-; SI-NEXT: v_readlane_b32 s20, v46, 43
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s31, 24
-; SI-NEXT: s_mov_b32 s56, s88
-; SI-NEXT: v_readlane_b32 s88, v46, 45
-; SI-NEXT: s_mov_b32 s27, s9
-; SI-NEXT: s_or_b32 s9, s19, s16
-; SI-NEXT: s_and_b32 s16, s20, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s88, 24
-; SI-NEXT: s_mov_b32 s91, s71
-; SI-NEXT: s_or_b32 s71, s19, s16
-; SI-NEXT: s_and_b32 s16, s41, 0xff
-; SI-NEXT: s_lshl_b32 s19, s61, 8
-; SI-NEXT: s_mov_b32 s78, s25
-; SI-NEXT: s_or_b32 s25, s16, s19
-; SI-NEXT: s_and_b32 s16, s89, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s59, 24
-; SI-NEXT: s_mov_b32 s39, s81
-; SI-NEXT: s_or_b32 s81, s19, s16
-; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s34, 24
-; SI-NEXT: s_mov_b32 s51, s80
-; SI-NEXT: s_or_b32 s80, s19, s16
-; SI-NEXT: s_and_b32 s16, s93, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s79, 24
-; SI-NEXT: v_readlane_b32 s97, v47, 30
-; SI-NEXT: s_mov_b32 s52, s82
-; SI-NEXT: s_or_b32 s82, s19, s16
-; SI-NEXT: s_and_b32 s16, s29, 0xff
-; SI-NEXT: s_lshl_b32 s19, s75, 8
-; SI-NEXT: s_mov_b32 s37, s4
-; SI-NEXT: s_mov_b32 s68, s24
-; SI-NEXT: s_or_b32 s24, s16, s19
-; SI-NEXT: s_and_b32 s16, s97, 0xff
-; SI-NEXT: v_readlane_b32 s4, v46, 47
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s19, s92, 24
-; SI-NEXT: s_lshl_b32 s94, s4, 16
-; SI-NEXT: v_readlane_b32 s4, v46, 48
-; SI-NEXT: s_mov_b32 s64, s96
-; SI-NEXT: s_mov_b32 s53, s83
-; SI-NEXT: s_mov_b32 s65, s22
-; SI-NEXT: s_mov_b32 s49, s76
-; SI-NEXT: s_mov_b32 s73, s92
-; SI-NEXT: s_mov_b32 s72, s6
-; SI-NEXT: s_or_b32 s83, s19, s16
-; SI-NEXT: s_lshl_b32 s22, s4, 16
-; SI-NEXT: s_lshl_b32 s19, s70, 16
-; SI-NEXT: s_lshl_b32 s16, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s5, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s7, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 s6, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s99, s84, 16
-; SI-NEXT: s_lshl_b32 s98, s85, 16
-; SI-NEXT: s_lshl_b32 s97, s86, 16
-; SI-NEXT: s_lshl_b32 s96, s87, 16
-; SI-NEXT: s_lshl_b32 s87, s23, 16
-; SI-NEXT: s_lshl_b32 s86, s26, 16
-; SI-NEXT: v_readlane_b32 s26, v46, 50
-; SI-NEXT: v_readlane_b32 s28, v46, 49
-; SI-NEXT: s_lshl_b32 s85, s25, 16
-; SI-NEXT: v_readlane_b32 s25, v46, 51
-; SI-NEXT: s_lshl_b32 s84, s24, 16
-; SI-NEXT: s_cbranch_execnz .LBB89_3
-; SI-NEXT: .LBB89_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s4, s29, 3
-; SI-NEXT: v_readlane_b32 s6, v47, 30
+; SI-NEXT: s_or_b32 s20, s7, s17
+; SI-NEXT: s_and_b32 s7, s78, 0xff
+; SI-NEXT: v_readlane_b32 s8, v47, 31
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s8, 24
+; SI-NEXT: s_or_b32 s66, s17, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 30
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s8, v47, 29
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s8, 24
+; SI-NEXT: s_mov_b32 s78, s65
+; SI-NEXT: s_or_b32 s65, s17, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 28
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s8, v47, 27
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s8, 24
+; SI-NEXT: s_mov_b32 s44, s67
+; SI-NEXT: s_or_b32 s67, s17, s7
+; SI-NEXT: v_readlane_b32 s7, v47, 26
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s17, s14, 8
+; SI-NEXT: s_mov_b32 s63, s23
+; SI-NEXT: s_or_b32 s23, s7, s17
+; SI-NEXT: s_and_b32 s7, s35, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s68, 24
+; SI-NEXT: s_mov_b32 s59, s69
+; SI-NEXT: s_or_b32 s69, s17, s7
+; SI-NEXT: s_and_b32 s7, s53, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s93, 24
+; SI-NEXT: s_or_b32 s10, s17, s7
+; SI-NEXT: s_and_b32 s7, s19, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s31, 24
+; SI-NEXT: s_or_b32 s8, s17, s7
+; SI-NEXT: s_and_b32 s7, s49, 0xff
+; SI-NEXT: s_lshl_b32 s17, s50, 8
+; SI-NEXT: s_mov_b32 s56, s26
+; SI-NEXT: s_or_b32 s26, s7, s17
+; SI-NEXT: v_readlane_b32 s7, v46, 57
+; SI-NEXT: s_mov_b32 s90, s7
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: v_readlane_b32 s12, v46, 56
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s17, s12, 24
+; SI-NEXT: s_mov_b32 s79, s71
+; SI-NEXT: s_or_b32 s71, s17, s7
+; SI-NEXT: v_readlane_b32 s7, v46, 58
+; SI-NEXT: s_lshl_b32 s17, s7, 16
+; SI-NEXT: v_readlane_b32 s7, v46, 59
+; SI-NEXT: s_mov_b32 s58, s18
+; SI-NEXT: s_mov_b32 s91, s87
+; SI-NEXT: s_mov_b32 vcc_lo, s35
+; SI-NEXT: s_mov_b32 s95, s68
+; SI-NEXT: s_mov_b32 s92, s53
+; SI-NEXT: s_mov_b32 s30, s93
+; SI-NEXT: s_mov_b32 s89, s19
+; SI-NEXT: s_mov_b32 s36, s31
+; SI-NEXT: s_mov_b32 s57, s49
+; SI-NEXT: s_mov_b32 s94, s50
+; SI-NEXT: s_mov_b32 s73, s12
+; SI-NEXT: s_lshl_b32 s18, s7, 16
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s93, s5, 16
+; SI-NEXT: s_lshl_b32 s99, s4, 16
+; SI-NEXT: s_lshl_b32 s98, s82, 16
+; SI-NEXT: s_lshl_b32 s97, s83, 16
+; SI-NEXT: s_lshl_b32 s96, s84, 16
+; SI-NEXT: s_lshl_b32 s87, s85, 16
+; SI-NEXT: s_lshl_b32 s86, s86, 16
+; SI-NEXT: s_lshl_b32 s85, s21, 16
+; SI-NEXT: s_lshl_b32 s84, s16, 16
+; SI-NEXT: s_lshl_b32 s83, s22, 16
+; SI-NEXT: s_lshl_b32 s82, s20, 16
+; SI-NEXT: s_lshl_b32 s81, s23, 16
+; SI-NEXT: v_readlane_b32 s23, v46, 61
+; SI-NEXT: v_readlane_b32 s24, v46, 60
+; SI-NEXT: s_lshl_b32 s80, s26, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB89_3
+; SI-NEXT: .LBB89_2:
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: s_mov_b32 s92, s53
+; SI-NEXT: s_mov_b32 s30, s93
+; SI-NEXT: s_mov_b32 vcc_lo, s35
+; SI-NEXT: s_mov_b32 s95, s68
+; SI-NEXT: s_mov_b32 s57, s31
+; SI-NEXT: s_mov_b32 s89, s14
+; SI-NEXT: s_mov_b32 s36, s19
+; SI-NEXT: s_mov_b32 s94, s50
+; SI-NEXT: s_mov_b32 s90, s49
+; SI-NEXT: s_mov_b32 s73, s4
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr17
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr18
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr97
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: .LBB89_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB89_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s4, s57, 3
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s75, 8
-; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: s_lshl_b32 s5, s94, 8
+; SI-NEXT: s_add_i32 s6, s90, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s5, s73, 24
@@ -145919,717 +146869,731 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; SI-NEXT: s_addk_i32 s4, 0x300
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_add_i32 s29, s40, 3
+; SI-NEXT: s_add_i32 s27, s92, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s29, 0xff
-; SI-NEXT: s_lshl_b32 s6, s34, 8
-; SI-NEXT: s_add_i32 s7, s93, 3
+; SI-NEXT: s_and_b32 s5, s27, 0xff
+; SI-NEXT: s_lshl_b32 s6, s30, 8
+; SI-NEXT: s_add_i32 s7, s89, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s79, 24
+; SI-NEXT: s_lshl_b32 s6, s36, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_addk_i32 s5, 0x300
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_add_i32 s40, s41, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s40, 0xff
-; SI-NEXT: s_lshl_b32 s7, s61, 8
-; SI-NEXT: s_add_i32 s10, s89, 3
+; SI-NEXT: v_readlane_b32 s6, v47, 26
+; SI-NEXT: s_add_i32 s14, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v46, 26
+; SI-NEXT: s_and_b32 s6, s14, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s16, vcc_lo, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: s_lshl_b32 s7, s59, 24
-; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_and_b32 s8, s16, 0xff
+; SI-NEXT: s_lshl_b32 s7, s95, 24
+; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_or_b32 s7, s7, s10
+; SI-NEXT: s_or_b32 s7, s7, s8
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_add_i32 s59, s49, 3
+; SI-NEXT: s_or_b32 s8, s7, s6
+; SI-NEXT: v_readlane_b32 s6, v47, 30
+; SI-NEXT: s_add_i32 s57, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 29
+; SI-NEXT: v_readlane_b32 s9, v47, 28
+; SI-NEXT: s_and_b32 s6, s57, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s28, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s59, 0xff
-; SI-NEXT: s_lshl_b32 s10, s31, 8
-; SI-NEXT: s_add_i32 s8, s20, 3
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: s_lshl_b32 s10, s88, 24
-; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: v_readlane_b32 s7, v47, 27
+; SI-NEXT: s_and_b32 s9, s28, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v46, 45
+; SI-NEXT: s_add_i32 s79, s7, 3
+; SI-NEXT: v_readlane_b32 s9, v46, 36
+; SI-NEXT: v_readlane_b32 s10, v46, 30
+; SI-NEXT: s_and_b32 s7, s79, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_add_i32 s41, s10, 3
+; SI-NEXT: s_or_b32 s7, s9, s7
+; SI-NEXT: v_readlane_b32 s9, v47, 31
+; SI-NEXT: s_and_b32 s12, s41, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_or_b32 s8, s10, s8
+; SI-NEXT: s_or_b32 s9, s9, s12
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: v_readlane_b32 s8, v46, 31
-; SI-NEXT: s_add_i32 s89, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 22
-; SI-NEXT: s_and_b32 s8, s89, 0xff
-; SI-NEXT: s_lshl_b32 s10, s9, 8
-; SI-NEXT: s_add_i32 s43, s47, 3
-; SI-NEXT: s_or_b32 s8, s10, s8
-; SI-NEXT: s_and_b32 s11, s43, 0xff
-; SI-NEXT: s_lshl_b32 s10, s95, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s10, s10, s8
-; SI-NEXT: v_readlane_b32 s8, v46, 36
-; SI-NEXT: s_add_i32 s31, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 32
-; SI-NEXT: s_and_b32 s8, s31, 0xff
-; SI-NEXT: s_lshl_b32 s11, s9, 8
-; SI-NEXT: v_readlane_b32 s9, v46, 20
-; SI-NEXT: s_or_b32 s8, s11, s8
-; SI-NEXT: s_lshl_b32 s11, s9, 24
-; SI-NEXT: v_readlane_b32 s9, v46, 21
-; SI-NEXT: s_add_i32 s60, s9, 3
-; SI-NEXT: s_and_b32 s14, s60, 0xff
+; SI-NEXT: s_or_b32 s7, s9, s7
+; SI-NEXT: v_readlane_b32 s9, v46, 50
+; SI-NEXT: s_add_i32 s95, s9, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 46
+; SI-NEXT: s_and_b32 s9, s95, 0xff
+; SI-NEXT: s_lshl_b32 s12, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 34
+; SI-NEXT: s_or_b32 s9, s12, s9
+; SI-NEXT: s_lshl_b32 s12, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 35
+; SI-NEXT: s_add_i32 s58, s10, 3
+; SI-NEXT: s_and_b32 s13, s58, 0xff
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: s_or_b32 s12, s12, s13
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_or_b32 s13, s12, s9
+; SI-NEXT: v_readlane_b32 s9, v46, 51
+; SI-NEXT: s_add_i32 vcc_lo, s9, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 48
+; SI-NEXT: s_and_b32 s9, vcc_lo, 0xff
+; SI-NEXT: s_lshl_b32 s12, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 38
+; SI-NEXT: s_or_b32 s9, s12, s9
+; SI-NEXT: s_lshl_b32 s12, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 41
+; SI-NEXT: s_add_i32 s74, s10, 3
+; SI-NEXT: s_and_b32 s14, s74, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s11, s11, s14
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s8
-; SI-NEXT: v_readlane_b32 s8, v46, 37
-; SI-NEXT: s_add_i32 s34, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 34
-; SI-NEXT: s_and_b32 s8, s34, 0xff
-; SI-NEXT: s_lshl_b32 s14, s9, 8
-; SI-NEXT: v_readlane_b32 s9, v46, 24
-; SI-NEXT: s_or_b32 s8, s14, s8
-; SI-NEXT: s_lshl_b32 s14, s9, 24
-; SI-NEXT: v_readlane_b32 s9, v46, 27
-; SI-NEXT: s_add_i32 s76, s9, 3
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: s_or_b32 s12, s12, s14
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_or_b32 s14, s12, s9
+; SI-NEXT: v_readlane_b32 s9, v46, 53
+; SI-NEXT: s_add_i32 s35, s9, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 52
+; SI-NEXT: s_and_b32 s9, s35, 0xff
+; SI-NEXT: s_lshl_b32 s12, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 39
+; SI-NEXT: s_or_b32 s9, s12, s9
+; SI-NEXT: s_lshl_b32 s12, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 42
+; SI-NEXT: s_add_i32 s76, s10, 3
; SI-NEXT: s_and_b32 s15, s76, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s14, s14, s8
-; SI-NEXT: v_readlane_b32 s8, v46, 39
-; SI-NEXT: s_add_i32 s39, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 38
-; SI-NEXT: s_and_b32 s8, s39, 0xff
-; SI-NEXT: s_lshl_b32 s15, s9, 8
-; SI-NEXT: v_readlane_b32 s9, v46, 25
-; SI-NEXT: s_or_b32 s8, s15, s8
-; SI-NEXT: s_lshl_b32 s15, s9, 24
-; SI-NEXT: v_readlane_b32 s9, v46, 28
-; SI-NEXT: s_add_i32 s78, s9, 3
-; SI-NEXT: s_and_b32 s16, s78, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s15, s15, s16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s15, s15, s8
-; SI-NEXT: v_readlane_b32 s8, v46, 35
-; SI-NEXT: s_add_i32 s30, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 33
-; SI-NEXT: s_and_b32 s8, s30, 0xff
-; SI-NEXT: s_lshl_b32 s16, s9, 8
-; SI-NEXT: v_readlane_b32 s9, v46, 30
-; SI-NEXT: s_or_b32 s8, s16, s8
-; SI-NEXT: s_lshl_b32 s16, s9, 24
-; SI-NEXT: v_readlane_b32 s9, v46, 29
-; SI-NEXT: s_add_i32 s79, s9, 3
-; SI-NEXT: s_and_b32 s17, s79, 0xff
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: s_or_b32 s12, s12, s15
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_or_b32 s16, s12, s9
+; SI-NEXT: v_readlane_b32 s9, v46, 49
+; SI-NEXT: s_add_i32 s94, s9, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 47
+; SI-NEXT: s_and_b32 s9, s94, 0xff
+; SI-NEXT: s_lshl_b32 s12, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 44
+; SI-NEXT: s_or_b32 s9, s12, s9
+; SI-NEXT: s_lshl_b32 s12, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 43
+; SI-NEXT: s_add_i32 s77, s10, 3
+; SI-NEXT: s_and_b32 s15, s77, 0xff
+; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: s_addk_i32 s9, 0x300
+; SI-NEXT: v_readlane_b32 s10, v46, 40
+; SI-NEXT: s_or_b32 s12, s12, s15
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_add_i32 s73, s10, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 37
+; SI-NEXT: s_or_b32 s9, s12, s9
+; SI-NEXT: s_and_b32 s12, s73, 0xff
+; SI-NEXT: s_lshl_b32 s15, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 33
+; SI-NEXT: s_or_b32 s12, s15, s12
+; SI-NEXT: s_lshl_b32 s15, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 32
+; SI-NEXT: s_add_i32 s46, s10, 3
+; SI-NEXT: s_and_b32 s17, s46, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s16, s16, s8
-; SI-NEXT: v_readlane_b32 s8, v46, 26
-; SI-NEXT: s_add_i32 s75, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 23
-; SI-NEXT: s_and_b32 s8, s75, 0xff
-; SI-NEXT: s_lshl_b32 s17, s9, 8
-; SI-NEXT: v_readlane_b32 s9, v46, 19
-; SI-NEXT: s_or_b32 s8, s17, s8
-; SI-NEXT: s_lshl_b32 s17, s9, 24
+; SI-NEXT: s_addk_i32 s12, 0x300
+; SI-NEXT: v_readlane_b32 s10, v46, 31
+; SI-NEXT: s_or_b32 s15, s15, s17
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_add_i32 s43, s10, 3
+; SI-NEXT: v_readlane_b32 s10, v46, 29
+; SI-NEXT: s_or_b32 s21, s15, s12
+; SI-NEXT: s_lshl_b32 s15, s10, 8
+; SI-NEXT: v_readlane_b32 s10, v46, 28
+; SI-NEXT: s_lshl_b32 s11, s10, 24
+; SI-NEXT: v_readlane_b32 s10, v46, 27
+; SI-NEXT: s_and_b32 s12, s43, 0xff
+; SI-NEXT: s_add_i32 s10, s10, 3
+; SI-NEXT: s_or_b32 s12, s15, s12
+; SI-NEXT: s_and_b32 s10, s10, 0xff
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_addk_i32 s12, 0x300
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s12, 0xffff
+; SI-NEXT: s_or_b32 s12, s10, s11
+; SI-NEXT: v_readlane_b32 s10, v46, 25
+; SI-NEXT: s_add_i32 s29, s10, 3
+; SI-NEXT: v_readlane_b32 s11, v46, 24
+; SI-NEXT: v_readlane_b32 s15, v46, 22
+; SI-NEXT: s_and_b32 s10, s29, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 8
+; SI-NEXT: s_add_i32 s19, s15, 3
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: v_readlane_b32 s11, v46, 23
+; SI-NEXT: s_and_b32 s15, s19, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: s_addk_i32 s10, 0x300
+; SI-NEXT: s_or_b32 s11, s11, s15
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_or_b32 s25, s11, s10
+; SI-NEXT: s_add_i32 s10, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v46, 21
+; SI-NEXT: s_add_i32 s11, s7, 0x3000000
+; SI-NEXT: s_add_i32 s19, s9, 0x3000000
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v46, 20
; SI-NEXT: v_readlane_b32 s9, v46, 18
-; SI-NEXT: s_add_i32 s56, s9, 3
-; SI-NEXT: s_and_b32 s18, s56, 0xff
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_add_i32 s45, s38, 3
-; SI-NEXT: s_or_b32 s23, s17, s8
-; SI-NEXT: s_and_b32 s8, s45, 0xff
-; SI-NEXT: s_lshl_b32 s17, s46, 8
-; SI-NEXT: s_add_i32 s12, s74, 3
-; SI-NEXT: s_or_b32 s8, s17, s8
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_lshl_b32 s13, s77, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_add_i32 s9, s35, 3
-; SI-NEXT: s_or_b32 s24, s12, s8
-; SI-NEXT: s_and_b32 s8, s9, 0xff
-; SI-NEXT: s_lshl_b32 s9, s90, 8
-; SI-NEXT: s_add_i32 s21, s36, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readlane_b32 s9, v46, 17
-; SI-NEXT: s_and_b32 s12, s21, 0xff
-; SI-NEXT: s_lshl_b32 s9, s9, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_addk_i32 s8, 0x300
-; SI-NEXT: s_or_b32 s9, s9, s12
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_or_b32 s27, s9, s8
-; SI-NEXT: s_add_i32 s8, s6, 0x3000000
-; SI-NEXT: v_readlane_b32 s6, v47, 32
-; SI-NEXT: s_add_i32 s9, s7, 0x3000000
-; SI-NEXT: s_add_i32 s12, s11, 0x3000000
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v46, 19
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s26, s7, s6
+; SI-NEXT: v_readlane_b32 s6, v46, 17
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: v_readlane_b32 s7, v46, 16
-; SI-NEXT: v_readlane_b32 s11, v46, 15
+; SI-NEXT: v_readlane_b32 s9, v46, 14
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 31
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: v_readlane_b32 s7, v46, 15
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s27, s7, s6
+; SI-NEXT: v_readlane_b32 s6, v46, 13
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v46, 12
+; SI-NEXT: v_readlane_b32 s9, v46, 10
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v46, 11
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s28, s7, s6
+; SI-NEXT: v_readlane_b32 s6, v46, 9
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v46, 8
+; SI-NEXT: v_readlane_b32 s9, v46, 6
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v46, 7
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s11
+; SI-NEXT: s_or_b32 s7, s7, s9
; SI-NEXT: s_or_b32 s29, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v47, 33
+; SI-NEXT: v_readlane_b32 s6, v46, 5
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 14
-; SI-NEXT: v_readlane_b32 s11, v46, 12
+; SI-NEXT: v_readlane_b32 s7, v46, 4
+; SI-NEXT: v_readlane_b32 s9, v46, 2
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 13
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: v_readlane_b32 s7, v46, 3
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s11
+; SI-NEXT: s_or_b32 s7, s7, s9
; SI-NEXT: s_or_b32 s40, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v46, 11
+; SI-NEXT: v_readlane_b32 s6, v46, 1
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 10
-; SI-NEXT: v_readlane_b32 s11, v46, 8
+; SI-NEXT: v_readlane_b32 s7, v46, 0
+; SI-NEXT: v_readlane_b32 s9, v47, 62
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 9
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: v_readlane_b32 s7, v47, 63
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s41, s7, s6
+; SI-NEXT: v_readlane_b32 s6, v47, 61
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 60
+; SI-NEXT: v_readlane_b32 s9, v47, 58
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 59
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s11
+; SI-NEXT: s_or_b32 s7, s7, s9
; SI-NEXT: s_or_b32 s42, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v46, 7
+; SI-NEXT: v_readlane_b32 s6, v47, 57
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 6
-; SI-NEXT: v_readlane_b32 s11, v46, 4
+; SI-NEXT: v_readlane_b32 s7, v47, 56
+; SI-NEXT: v_readlane_b32 s9, v47, 54
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 5
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: v_readlane_b32 s7, v47, 55
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s11
-; SI-NEXT: s_or_b32 s43, s7, s6
-; SI-NEXT: v_readlane_b32 s6, v46, 3
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s43, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v47, 7
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 2
-; SI-NEXT: v_readlane_b32 s11, v46, 0
+; SI-NEXT: v_readlane_b32 s7, v47, 53
+; SI-NEXT: v_readlane_b32 s9, v47, 51
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
-; SI-NEXT: s_add_i32 s11, s11, 3
+; SI-NEXT: s_add_i32 s9, s9, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 1
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: v_readlane_b32 s7, v47, 52
+; SI-NEXT: s_and_b32 s9, s9, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s11
+; SI-NEXT: s_or_b32 s7, s7, s9
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v47, 63
-; SI-NEXT: s_add_i32 s13, s14, 0x3000000
+; SI-NEXT: s_add_i32 s44, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v47, 11
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 10
+; SI-NEXT: v_readlane_b32 s9, v47, 9
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 8
+; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s9, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v47, 15
+; SI-NEXT: s_add_i32 s22, s12, 0x3000000
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 14
+; SI-NEXT: v_readlane_b32 s12, v47, 13
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 12
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s15, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v47, 19
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 18
+; SI-NEXT: v_readlane_b32 s12, v47, 17
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 16
+; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s12, s6, 0x3000000
+; SI-NEXT: v_readlane_b32 s6, v47, 23
+; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: v_readlane_b32 s7, v47, 22
+; SI-NEXT: v_readlane_b32 s17, v47, 21
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 20
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: v_readlane_b32 s7, v47, 45
; SI-NEXT: s_add_i32 s7, s7, 3
-; SI-NEXT: v_readlane_b32 s11, v47, 62
-; SI-NEXT: v_readlane_b32 s14, v47, 60
+; SI-NEXT: v_readlane_b32 s17, v47, 43
+; SI-NEXT: v_readlane_b32 s18, v47, 25
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 8
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: v_readlane_b32 s11, v47, 61
-; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s7, s17, s7
+; SI-NEXT: v_readlane_b32 s17, v47, 24
+; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s11, s11, 24
-; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s14
-; SI-NEXT: s_or_b32 s44, s11, s7
-; SI-NEXT: v_readlane_b32 s7, v47, 59
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s7, s17, s7
+; SI-NEXT: s_add_i32 s20, s7, 0x3000000
+; SI-NEXT: v_readlane_b32 s7, v47, 46
; SI-NEXT: s_add_i32 s7, s7, 3
-; SI-NEXT: v_readlane_b32 s11, v47, 58
-; SI-NEXT: v_readlane_b32 s14, v47, 56
+; SI-NEXT: v_readlane_b32 s17, v47, 44
+; SI-NEXT: v_readlane_b32 s18, v47, 40
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 8
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: v_readlane_b32 s11, v47, 57
-; SI-NEXT: s_and_b32 s14, s14, 0xff
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_or_b32 s7, s17, s7
+; SI-NEXT: v_readlane_b32 s17, v47, 39
+; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s11, s11, 24
-; SI-NEXT: s_lshl_b32 s14, s14, 16
+; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s14
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: v_readlane_b32 s11, v47, 55
-; SI-NEXT: s_add_i32 s21, s16, 0x3000000
-; SI-NEXT: s_add_i32 s11, s11, 3
-; SI-NEXT: v_readlane_b32 s14, v47, 54
-; SI-NEXT: v_readlane_b32 s16, v47, 52
-; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s14, s14, 8
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: v_readlane_b32 s14, v47, 53
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_addk_i32 s11, 0x300
-; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: s_add_i32 s45, s11, 0x3000000
-; SI-NEXT: v_readlane_b32 s11, v47, 13
-; SI-NEXT: s_add_i32 s11, s11, 3
-; SI-NEXT: v_readlane_b32 s14, v47, 51
-; SI-NEXT: v_readlane_b32 s16, v47, 49
-; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s14, s14, 8
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: v_readlane_b32 s14, v47, 50
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_addk_i32 s11, 0x300
-; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: s_add_i32 s18, s11, 0x3000000
-; SI-NEXT: v_readlane_b32 s11, v47, 17
-; SI-NEXT: s_add_i32 s11, s11, 3
-; SI-NEXT: v_readlane_b32 s14, v47, 16
-; SI-NEXT: v_readlane_b32 s16, v47, 15
-; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s14, s14, 8
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: v_readlane_b32 s14, v47, 14
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_addk_i32 s11, 0x300
-; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: s_or_b32 s11, s14, s11
-; SI-NEXT: v_readlane_b32 s14, v47, 21
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: v_readlane_b32 s16, v47, 20
-; SI-NEXT: v_readlane_b32 s17, v47, 19
-; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 8
+; SI-NEXT: s_or_b32 s17, s17, s18
+; SI-NEXT: s_or_b32 s7, s17, s7
+; SI-NEXT: v_readlane_b32 s17, v47, 49
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s14, s16, s14
-; SI-NEXT: v_readlane_b32 s16, v47, 18
+; SI-NEXT: v_readlane_b32 s18, v47, 47
+; SI-NEXT: v_readlane_b32 s23, v47, 42
; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: s_addk_i32 s14, 0x300
-; SI-NEXT: s_lshl_b32 s16, s16, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_or_b32 s14, s16, s14
-; SI-NEXT: s_add_i32 s17, s14, 0x3000000
-; SI-NEXT: v_readlane_b32 s14, v47, 25
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: v_readlane_b32 s16, v47, 24
-; SI-NEXT: v_readlane_b32 s19, v47, 23
-; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s14, s16, s14
-; SI-NEXT: v_readlane_b32 s16, v47, 22
-; SI-NEXT: s_and_b32 s19, s19, 0xff
-; SI-NEXT: s_addk_i32 s14, 0x300
-; SI-NEXT: s_lshl_b32 s16, s16, 24
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_or_b32 s16, s16, s19
-; SI-NEXT: s_or_b32 s14, s16, s14
-; SI-NEXT: v_readlane_b32 s16, v47, 29
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s19, v47, 28
-; SI-NEXT: v_readlane_b32 s20, v47, 27
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s19, s19, 8
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_or_b32 s16, s19, s16
-; SI-NEXT: v_readlane_b32 s19, v47, 26
-; SI-NEXT: s_and_b32 s20, s20, 0xff
-; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s19, s19, 24
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: s_or_b32 s16, s19, s16
-; SI-NEXT: v_readlane_b32 s19, v47, 43
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: v_readlane_b32 s20, v47, 41
-; SI-NEXT: v_readlane_b32 s22, v47, 36
-; SI-NEXT: s_and_b32 s19, s19, 0xff
-; SI-NEXT: s_lshl_b32 s20, s20, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s19, s20, s19
-; SI-NEXT: v_readlane_b32 s20, v47, 35
-; SI-NEXT: s_and_b32 s22, s22, 0xff
-; SI-NEXT: s_addk_i32 s19, 0x300
-; SI-NEXT: s_lshl_b32 s20, s20, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_or_b32 s20, s20, s22
-; SI-NEXT: s_or_b32 s19, s20, s19
-; SI-NEXT: s_add_i32 s25, s19, 0x3000000
-; SI-NEXT: v_readlane_b32 s19, v47, 44
-; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: v_readlane_b32 s20, v47, 42
-; SI-NEXT: v_readlane_b32 s22, v47, 38
-; SI-NEXT: s_and_b32 s19, s19, 0xff
-; SI-NEXT: s_lshl_b32 s20, s20, 8
-; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s19, s20, s19
-; SI-NEXT: v_readlane_b32 s20, v47, 37
-; SI-NEXT: s_and_b32 s22, s22, 0xff
-; SI-NEXT: s_addk_i32 s19, 0x300
-; SI-NEXT: s_lshl_b32 s20, s20, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_or_b32 s20, s20, s22
-; SI-NEXT: s_or_b32 s19, s20, s19
-; SI-NEXT: v_readlane_b32 s20, v47, 47
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s22, v47, 45
-; SI-NEXT: v_readlane_b32 s26, v47, 40
-; SI-NEXT: s_and_b32 s20, s20, 0xff
-; SI-NEXT: s_lshl_b32 s22, s22, 8
-; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_or_b32 s20, s22, s20
-; SI-NEXT: v_readlane_b32 s22, v47, 39
-; SI-NEXT: s_and_b32 s26, s26, 0xff
-; SI-NEXT: s_addk_i32 s20, 0x300
-; SI-NEXT: s_lshl_b32 s22, s22, 24
-; SI-NEXT: s_lshl_b32 s26, s26, 16
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_or_b32 s22, s22, s26
-; SI-NEXT: s_or_b32 s20, s22, s20
-; SI-NEXT: s_add_i32 s26, s20, 0x3000000
-; SI-NEXT: v_readlane_b32 s20, v47, 1
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s22, v47, 0
-; SI-NEXT: v_readlane_b32 s28, v47, 48
-; SI-NEXT: s_and_b32 s20, s20, 0xff
-; SI-NEXT: s_lshl_b32 s22, s22, 8
-; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_or_b32 s20, s22, s20
-; SI-NEXT: v_readlane_b32 s22, v47, 46
-; SI-NEXT: s_and_b32 s28, s28, 0xff
-; SI-NEXT: s_addk_i32 s20, 0x300
-; SI-NEXT: s_lshl_b32 s22, s22, 24
-; SI-NEXT: s_lshl_b32 s28, s28, 16
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_or_b32 s22, s22, s28
-; SI-NEXT: s_or_b32 s20, s22, s20
-; SI-NEXT: s_add_i32 s22, s20, 0x3000000
-; SI-NEXT: v_readlane_b32 s20, v47, 5
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s28, v47, 4
-; SI-NEXT: v_readlane_b32 s41, v47, 3
-; SI-NEXT: s_and_b32 s20, s20, 0xff
-; SI-NEXT: s_lshl_b32 s28, s28, 8
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_or_b32 s20, s28, s20
-; SI-NEXT: v_readlane_b32 s28, v47, 2
-; SI-NEXT: s_and_b32 s41, s41, 0xff
-; SI-NEXT: s_addk_i32 s20, 0x300
-; SI-NEXT: s_lshl_b32 s28, s28, 24
-; SI-NEXT: s_lshl_b32 s41, s41, 16
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_or_b32 s28, s28, s41
-; SI-NEXT: s_or_b32 s20, s28, s20
-; SI-NEXT: s_add_i32 s28, s20, 0x3000000
-; SI-NEXT: v_readlane_b32 s20, v47, 9
-; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s41, v47, 8
-; SI-NEXT: v_readlane_b32 s46, v47, 7
-; SI-NEXT: s_and_b32 s20, s20, 0xff
-; SI-NEXT: s_lshl_b32 s41, s41, 8
+; SI-NEXT: s_lshl_b32 s18, s18, 8
+; SI-NEXT: s_add_i32 s23, s23, 3
+; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v47, 41
+; SI-NEXT: s_and_b32 s23, s23, 0xff
+; SI-NEXT: s_addk_i32 s17, 0x300
+; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s23, s23, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_or_b32 s18, s18, s23
+; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: s_add_i32 s23, s17, 0x3000000
+; SI-NEXT: v_readlane_b32 s17, v47, 1
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: v_readlane_b32 s18, v47, 0
+; SI-NEXT: v_readlane_b32 s24, v47, 50
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: s_lshl_b32 s18, s18, 8
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: v_readlane_b32 s18, v47, 48
+; SI-NEXT: s_and_b32 s24, s24, 0xff
+; SI-NEXT: s_addk_i32 s17, 0x300
+; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_or_b32 s18, s18, s24
+; SI-NEXT: s_or_b32 s17, s18, s17
+; SI-NEXT: s_add_i32 s18, s17, 0x3000000
+; SI-NEXT: v_readlane_b32 s17, v47, 5
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: v_readlane_b32 s24, v47, 4
+; SI-NEXT: v_readlane_b32 s45, v47, 3
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: s_lshl_b32 s24, s24, 8
+; SI-NEXT: s_add_i32 s45, s45, 3
+; SI-NEXT: s_or_b32 s17, s24, s17
+; SI-NEXT: v_readlane_b32 s24, v47, 2
+; SI-NEXT: s_and_b32 s45, s45, 0xff
+; SI-NEXT: s_addk_i32 s17, 0x300
+; SI-NEXT: s_lshl_b32 s24, s24, 24
+; SI-NEXT: s_lshl_b32 s45, s45, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_or_b32 s24, s24, s45
+; SI-NEXT: s_or_b32 s17, s24, s17
+; SI-NEXT: s_add_i32 s24, s17, 0x3000000
+; SI-NEXT: v_readlane_b32 s17, v47, 38
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: v_readlane_b32 s45, v47, 37
+; SI-NEXT: v_readlane_b32 s46, v47, 36
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: s_lshl_b32 s45, s45, 8
; SI-NEXT: s_add_i32 s46, s46, 3
-; SI-NEXT: s_or_b32 s20, s41, s20
-; SI-NEXT: v_readlane_b32 s41, v47, 6
+; SI-NEXT: s_or_b32 s17, s45, s17
+; SI-NEXT: v_readlane_b32 s45, v47, 6
; SI-NEXT: s_and_b32 s46, s46, 0xff
-; SI-NEXT: s_addk_i32 s20, 0x300
-; SI-NEXT: s_lshl_b32 s41, s41, 24
+; SI-NEXT: s_addk_i32 s17, 0x300
+; SI-NEXT: s_lshl_b32 s45, s45, 24
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_or_b32 s41, s41, s46
-; SI-NEXT: s_or_b32 s20, s41, s20
-; SI-NEXT: v_readlane_b32 s41, v47, 34
-; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: v_readlane_b32 s46, v47, 12
-; SI-NEXT: v_readlane_b32 s47, v47, 11
-; SI-NEXT: s_and_b32 s41, s41, 0xff
+; SI-NEXT: s_and_b32 s17, s17, 0xffff
+; SI-NEXT: s_or_b32 s45, s45, s46
+; SI-NEXT: s_or_b32 s17, s45, s17
+; SI-NEXT: v_readlane_b32 s45, v47, 35
+; SI-NEXT: s_add_i32 s45, s45, 3
+; SI-NEXT: v_readlane_b32 s46, v47, 34
+; SI-NEXT: v_readlane_b32 s47, v47, 32
+; SI-NEXT: s_and_b32 s45, s45, 0xff
; SI-NEXT: s_lshl_b32 s46, s46, 8
; SI-NEXT: s_add_i32 s47, s47, 3
-; SI-NEXT: s_or_b32 s41, s46, s41
-; SI-NEXT: v_readlane_b32 s46, v47, 10
+; SI-NEXT: s_or_b32 s45, s46, s45
+; SI-NEXT: v_readlane_b32 s46, v47, 33
; SI-NEXT: s_and_b32 s47, s47, 0xff
-; SI-NEXT: s_addk_i32 s41, 0x300
+; SI-NEXT: s_addk_i32 s45, 0x300
; SI-NEXT: s_lshl_b32 s46, s46, 24
; SI-NEXT: s_lshl_b32 s47, s47, 16
-; SI-NEXT: s_and_b32 s41, s41, 0xffff
+; SI-NEXT: s_and_b32 s45, s45, 0xffff
; SI-NEXT: s_or_b32 s46, s46, s47
-; SI-NEXT: s_or_b32 s41, s46, s41
+; SI-NEXT: s_or_b32 s45, s46, s45
+; SI-NEXT: s_add_i32 s45, s45, 0x3000000
; SI-NEXT: s_add_i32 s41, s41, 0x3000000
-; SI-NEXT: s_add_i32 s40, s40, 0x3000000
-; SI-NEXT: s_add_i32 s44, s44, 0x3000000
-; SI-NEXT: s_add_i32 s20, s20, 0x3000000
-; SI-NEXT: s_and_b32 s57, s41, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s41, 16
-; SI-NEXT: s_and_b32 s41, s28, 0xffff0000
+; SI-NEXT: s_and_b32 s46, s45, 0xffff0000
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 s10, s10, 0x3000000
-; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: s_add_i32 s23, s23, 0x3000000
-; SI-NEXT: s_add_i32 s24, s24, 0x3000000
+; SI-NEXT: s_add_i32 s8, s8, 0x3000000
+; SI-NEXT: s_add_i32 s13, s13, 0x3000000
+; SI-NEXT: s_add_i32 s14, s14, 0x3000000
+; SI-NEXT: s_add_i32 s16, s16, 0x3000000
+; SI-NEXT: s_add_i32 s21, s21, 0x3000000
+; SI-NEXT: s_add_i32 s25, s25, 0x3000000
+; SI-NEXT: s_add_i32 s26, s26, 0x3000000
; SI-NEXT: s_add_i32 s27, s27, 0x3000000
+; SI-NEXT: s_add_i32 s28, s28, 0x3000000
; SI-NEXT: s_add_i32 s29, s29, 0x3000000
+; SI-NEXT: s_add_i32 s40, s40, 0x3000000
; SI-NEXT: s_add_i32 s42, s42, 0x3000000
-; SI-NEXT: s_add_i32 s43, s43, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: s_add_i32 s14, s14, 0x3000000
-; SI-NEXT: s_add_i32 s16, s16, 0x3000000
-; SI-NEXT: s_add_i32 s19, s19, 0x3000000
-; SI-NEXT: s_and_b32 s60, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s94, s20, 16
-; SI-NEXT: v_writelane_b32 v46, s41, 40
-; SI-NEXT: s_lshl_b32 s28, s28, 16
-; SI-NEXT: s_lshl_b32 s20, s44, 16
-; SI-NEXT: s_and_b32 s66, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: v_writelane_b32 v46, s28, 41
-; SI-NEXT: s_and_b32 s28, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_and_b32 s62, s26, 0xffff0000
-; SI-NEXT: s_lshl_b32 s26, s26, 16
-; SI-NEXT: s_and_b32 s63, s19, 0xffff0000
-; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s72, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_and_b32 s78, s16, 0xffff0000
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_and_b32 s48, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s58, s14, 16
-; SI-NEXT: s_and_b32 s37, s17, 0xffff0000
+; SI-NEXT: s_add_i32 s17, s17, 0x3000000
+; SI-NEXT: v_writelane_b32 v46, s46, 54
+; SI-NEXT: s_and_b32 s60, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_lshl_b32 s57, s41, 16
+; SI-NEXT: s_lshl_b32 s46, s45, 16
+; SI-NEXT: s_and_b32 s58, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s14, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s50, s11, 16
-; SI-NEXT: s_and_b32 s51, s18, 0xffff0000
+; SI-NEXT: v_writelane_b32 v46, s24, 55
+; SI-NEXT: s_and_b32 s24, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_and_b32 s39, s45, 0xffff0000
-; SI-NEXT: s_lshl_b32 s91, s45, 16
-; SI-NEXT: s_and_b32 s52, s7, 0xffff0000
+; SI-NEXT: s_and_b32 s61, s23, 0xffff0000
+; SI-NEXT: s_lshl_b32 s23, s23, 16
+; SI-NEXT: s_and_b32 s62, s7, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s53, s44, 0xffff0000
-; SI-NEXT: s_and_b32 s55, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s64, s43, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s43, 16
-; SI-NEXT: s_and_b32 s65, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s99, s42, 16
-; SI-NEXT: s_and_b32 s68, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s98, s29, 16
-; SI-NEXT: s_and_b32 s11, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s27, s27, 16
-; SI-NEXT: s_and_b32 s43, s24, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s24, 16
-; SI-NEXT: s_and_b32 s42, s23, 0xffff0000
-; SI-NEXT: s_lshl_b32 s44, s23, 16
-; SI-NEXT: s_and_b32 s45, s21, 0xffff0000
-; SI-NEXT: s_lshl_b32 s96, s21, 16
-; SI-NEXT: s_and_b32 s30, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_and_b32 s67, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s13, 16
-; SI-NEXT: s_and_b32 s21, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s12, 16
-; SI-NEXT: s_and_b32 s12, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s86, s10, 16
-; SI-NEXT: s_mov_b32 s10, s20
-; SI-NEXT: s_and_b32 s71, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s81, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s8, 16
-; SI-NEXT: s_mov_b32 s8, s40
-; SI-NEXT: s_and_b32 s82, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s80, s5, 16
-; SI-NEXT: s_and_b32 s83, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s84, s4, 16
-; SI-NEXT: .LBB89_3: ; %end
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
+; SI-NEXT: s_and_b32 s63, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s70, s20, 16
+; SI-NEXT: s_and_b32 s56, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s93, s6, 16
+; SI-NEXT: s_and_b32 s59, s12, 0xffff0000
+; SI-NEXT: s_lshl_b32 s6, s12, 16
+; SI-NEXT: s_and_b32 s77, s15, 0xffff0000
+; SI-NEXT: s_lshl_b32 s99, s15, 16
+; SI-NEXT: s_and_b32 s78, s9, 0xffff0000
+; SI-NEXT: s_lshl_b32 s75, s9, 16
+; SI-NEXT: s_and_b32 s79, s44, 0xffff0000
+; SI-NEXT: s_lshl_b32 s98, s44, 16
+; SI-NEXT: s_and_b32 s88, s43, 0xffff0000
+; SI-NEXT: s_lshl_b32 s44, s43, 16
+; SI-NEXT: s_and_b32 s91, s42, 0xffff0000
+; SI-NEXT: s_lshl_b32 s97, s42, 16
+; SI-NEXT: s_and_b32 s9, s41, 0xffff0000
+; SI-NEXT: s_and_b32 s15, s40, 0xffff0000
+; SI-NEXT: s_lshl_b32 s96, s40, 16
+; SI-NEXT: s_and_b32 s40, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s29, s29, 16
+; SI-NEXT: s_and_b32 s42, s28, 0xffff0000
+; SI-NEXT: s_lshl_b32 s87, s28, 16
+; SI-NEXT: s_and_b32 s28, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 s41, s27, 16
+; SI-NEXT: s_and_b32 s47, s26, 0xffff0000
+; SI-NEXT: s_lshl_b32 s86, s26, 16
+; SI-NEXT: s_and_b32 s27, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s43, s25, 16
+; SI-NEXT: s_and_b32 s72, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s85, s22, 16
+; SI-NEXT: s_and_b32 s74, s21, 0xffff0000
+; SI-NEXT: s_lshl_b32 s25, s21, 16
+; SI-NEXT: s_and_b32 s52, s19, 0xffff0000
+; SI-NEXT: s_lshl_b32 s84, s19, 16
+; SI-NEXT: s_and_b32 s76, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s51, s16, 16
+; SI-NEXT: s_and_b32 s55, s14, 0xffff0000
+; SI-NEXT: s_lshl_b32 s83, s14, 16
+; SI-NEXT: s_and_b32 s45, s13, 0xffff0000
+; SI-NEXT: s_lshl_b32 s54, s13, 16
+; SI-NEXT: s_and_b32 s66, s11, 0xffff0000
+; SI-NEXT: s_lshl_b32 s82, s11, 16
+; SI-NEXT: s_mov_b32 s11, s57
+; SI-NEXT: s_and_b32 s67, s10, 0xffff0000
+; SI-NEXT: s_lshl_b32 s65, s10, 16
+; SI-NEXT: s_and_b32 s69, s8, 0xffff0000
+; SI-NEXT: s_lshl_b32 s81, s8, 16
+; SI-NEXT: s_and_b32 s8, s5, 0xffff0000
+; SI-NEXT: s_lshl_b32 s10, s5, 16
+; SI-NEXT: s_and_b32 s71, s4, 0xffff0000
+; SI-NEXT: s_lshl_b32 s80, s4, 16
+; SI-NEXT: .LBB89_5: ; %end
+; SI-NEXT: v_readlane_b32 s4, v46, 54
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s58
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s17
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_readlane_b32 s4, v46, 40
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; SI-NEXT: v_readlane_b32 s4, v46, 41
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s60
+; SI-NEXT: v_readlane_b32 s4, v46, 55
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s4
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s28
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s24
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s18
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s61
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s26
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s23
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s63
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s62
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s63
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s70
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s56
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s93
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s48
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s59
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s99
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s78
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s50
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s75
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s51
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s79
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s98
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s39
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s88
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s44
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s52
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s91
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s97
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s53
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s10
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s11
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s55
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s96
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s64
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s40
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s54
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s29
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s65
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s99
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s87
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s66
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s28
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s41
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s68
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s47
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s98
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s86
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s11
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
; SI-NEXT: v_mul_f32_e64 v20, 1.0, s27
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s43
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s72
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s97
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s85
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s42
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s74
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s44
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s25
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s52
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s96
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s84
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s76
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s15
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s51
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s67
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s55
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s87
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s83
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s45
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s54
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s12
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s66
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s86
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s82
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s71
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s67
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s65
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s81
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s69
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s85
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s81
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s82
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s80
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s10
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s83
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s71
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s84
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s80
; SI-NEXT: v_readlane_b32 s30, v45, 34
; SI-NEXT: v_lshr_b64 v[31:32], v[31:32], 16
; SI-NEXT: v_readlane_b32 s31, v45, 35
@@ -146674,76 +147638,6 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB89_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_mov_b32 s49, s76
-; SI-NEXT: s_mov_b32 s73, s92
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr17
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: s_branch .LBB89_2
;
; VI-LABEL: bitcast_v128i8_to_v64bf16_scalar:
; VI: ; %bb.0:
@@ -146960,7 +147854,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; VI-NEXT: s_cbranch_scc0 .LBB89_4
+; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
@@ -147180,12 +148074,32 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: v_mov_b32_e32 v35, v43
; VI-NEXT: v_perm_b32 v43, v44, v43, s4
; VI-NEXT: v_or_b32_e32 v31, v43, v31
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: v_mov_b32_e32 v43, v39
; VI-NEXT: v_mov_b32_e32 v39, v34
; VI-NEXT: v_mov_b32_e32 v34, v52
; VI-NEXT: v_mov_b32_e32 v52, v45
-; VI-NEXT: s_cbranch_execnz .LBB89_3
-; VI-NEXT: .LBB89_2: ; %cmp.true
+; VI-NEXT: s_branch .LBB89_3
+; VI-NEXT: .LBB89_2:
+; VI-NEXT: v_mov_b32_e32 v43, v39
+; VI-NEXT: v_mov_b32_e32 v52, v34
+; VI-NEXT: v_mov_b32_e32 v39, v33
+; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v34, v53
+; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v62, v2
+; VI-NEXT: v_mov_b32_e32 v35, v1
+; VI-NEXT: v_mov_b32_e32 v55, v57
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT: .LBB89_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB89_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -147600,7 +148514,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: v_or_b32_sdwa v27, v46, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v18, vcc, 0x3000000, v18
; VI-NEXT: v_add_u32_e32 v27, vcc, 0x3000000, v27
-; VI-NEXT: .LBB89_3: ; %end
+; VI-NEXT: .LBB89_5: ; %end
; VI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
@@ -147619,20 +148533,6 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB89_4:
-; VI-NEXT: v_mov_b32_e32 v43, v39
-; VI-NEXT: v_mov_b32_e32 v52, v34
-; VI-NEXT: v_mov_b32_e32 v39, v33
-; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v34, v53
-; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v62, v2
-; VI-NEXT: v_mov_b32_e32 v35, v1
-; VI-NEXT: v_mov_b32_e32 v55, v57
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT: s_branch .LBB89_2
;
; GFX9-LABEL: bitcast_v128i8_to_v64bf16_scalar:
; GFX9: ; %bb.0:
@@ -148124,8 +149024,10 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB89_3: ; %Flow
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB89_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
@@ -148677,7 +149579,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB89_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB89_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v86
@@ -148829,9 +149731,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB89_3
-; GFX11-TRUE16-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB89_3
+; GFX11-TRUE16-NEXT: .LBB89_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB89_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
@@ -149060,16 +149971,12 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
-; GFX11-TRUE16-NEXT: .LBB89_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB89_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB89_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB89_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v64bf16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -149193,7 +150100,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB89_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB89_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s74, 0xff
@@ -149384,9 +150291,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v40
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB89_3
-; GFX11-FAKE16-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB89_3
+; GFX11-FAKE16-NEXT: .LBB89_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB89_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v182
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v180
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v178
@@ -149647,14 +150563,10 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v29, v150, 16, v29
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v30, v30, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v35
-; GFX11-FAKE16-NEXT: .LBB89_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB89_5: ; %end
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:320 ; 4-byte Folded Reload
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB89_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB89_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -157162,7 +158074,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v24, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s28
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s19
; SI-NEXT: v_mul_f32_e64 v27, 1.0, s27
; SI-NEXT: v_mul_f32_e64 v29, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s26
@@ -157173,24 +158085,24 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v30, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v45, 1.0, s14
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s12
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s13
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s11
; SI-NEXT: v_mul_f32_e64 v56, 1.0, s8
; SI-NEXT: v_mul_f32_e64 v34, 1.0, s9
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v40, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v22, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v49, 1.0, s35
; SI-NEXT: v_mul_f32_e64 v42, 1.0, s34
; SI-NEXT: v_mul_f32_e64 v48, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, vcc_lo
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, vcc_hi
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, vcc_lo
; SI-NEXT: v_mul_f32_e64 v53, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v44, 1.0, s93
; SI-NEXT: v_mul_f32_e64 v47, 1.0, s92
; SI-NEXT: v_mul_f32_e64 v41, 1.0, s91
@@ -157198,19 +158110,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v57, 1.0, s89
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s88
; SI-NEXT: v_mul_f32_e64 v46, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s78
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s77
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s76
; SI-NEXT: v_mul_f32_e64 v58, 1.0, s75
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v43, 1.0, s74
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s72
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s61
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s60
; SI-NEXT: v_mul_f32_e64 v12, 1.0, s59
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s58
; SI-NEXT: v_mul_f32_e64 v18, 1.0, s47
; SI-NEXT: v_mul_f32_e64 v39, 1.0, s45
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
@@ -157222,7 +158134,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
@@ -157240,8 +158152,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB91_2
@@ -157249,15 +158161,15 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: s_lshr_b32 s7, s4, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v24
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v25
; SI-NEXT: s_lshr_b64 s[86:87], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: s_lshr_b32 s65, s4, 16
+; SI-NEXT: s_lshr_b32 s49, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v27
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v29
@@ -157270,51 +158182,56 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s4, v31
; SI-NEXT: s_lshr_b64 s[66:67], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: s_lshr_b32 s93, s4, 16
+; SI-NEXT: s_lshr_b32 s75, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v45
+; SI-NEXT: v_mov_b32_e32 v52, v13
+; SI-NEXT: v_mov_b32_e32 v13, v23
+; SI-NEXT: v_mov_b32_e32 v23, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v60
+; SI-NEXT: v_mov_b32_e32 v60, v36
+; SI-NEXT: v_mov_b32_e32 v15, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v45, v38
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b64 s[52:53], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v20
+; SI-NEXT: v_readfirstlane_b32 s4, v19
; SI-NEXT: s_lshr_b32 s39, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v35
-; SI-NEXT: v_mov_b32_e32 v35, v49
+; SI-NEXT: v_mov_b32_e32 v35, v51
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v35
; SI-NEXT: s_lshr_b64 s[34:35], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v56
-; SI-NEXT: v_mov_b32_e32 v52, v54
-; SI-NEXT: s_lshr_b32 s91, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v52
-; SI-NEXT: v_mov_b32_e32 v45, v40
+; SI-NEXT: v_mov_b32_e32 v51, v54
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: s_lshr_b32 s15, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v51
+; SI-NEXT: v_mov_b32_e32 v4, v40
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v45
+; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_mov_b32_e32 v21, v22
-; SI-NEXT: v_readfirstlane_b32 s6, v4
; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: v_mov_b32_e32 v4, v38
-; SI-NEXT: s_lshr_b32 s57, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_mov_b32_e32 v40, v49
+; SI-NEXT: s_lshr_b32 s43, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v40
; SI-NEXT: v_mov_b32_e32 v54, v42
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v54
-; SI-NEXT: v_mov_b32_e32 v22, v48
; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v22
-; SI-NEXT: v_mov_b32_e32 v40, v51
-; SI-NEXT: s_lshr_b32 s89, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v40
+; SI-NEXT: v_readfirstlane_b32 s4, v48
; SI-NEXT: v_mov_b32_e32 v42, v50
-; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: s_lshr_b32 s89, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v42
-; SI-NEXT: v_mov_b32_e32 v49, v2
-; SI-NEXT: v_mov_b32_e32 v2, v11
+; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_lshr_b64 s[76:77], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v53
-; SI-NEXT: v_mov_b32_e32 v11, v44
; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v11
+; SI-NEXT: v_readfirstlane_b32 s4, v44
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v47
; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
@@ -157329,282 +158246,279 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: v_mov_b32_e32 v44, v32
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v58
-; SI-NEXT: v_mov_b32_e32 v32, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s29, s4, 16
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mov_b32_e32 v1, v9
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v9, v6
; SI-NEXT: v_readfirstlane_b32 s4, v6
-; SI-NEXT: v_mov_b32_e32 v6, v36
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v6
-; SI-NEXT: v_mov_b32_e32 v3, v23
+; SI-NEXT: v_readfirstlane_b32 s4, v13
; SI-NEXT: s_lshr_b64 s[24:25], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: v_mov_b32_e32 v23, v43
; SI-NEXT: s_lshr_b32 s23, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v23
+; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v32
+; SI-NEXT: v_readfirstlane_b32 s4, v23
; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
-; SI-NEXT: s_mov_b32 s9, s96
-; SI-NEXT: v_readfirstlane_b32 s90, v34
-; SI-NEXT: s_lshr_b64 s[82:83], s[90:91], 16
-; SI-NEXT: v_readfirstlane_b32 s64, v19
-; SI-NEXT: s_lshr_b64 s[84:85], s[64:65], 16
+; SI-NEXT: v_readfirstlane_b32 s14, v34
+; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
+; SI-NEXT: v_readfirstlane_b32 s72, v60
+; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[98:99], vcc, 16
+; SI-NEXT: s_mov_b32 s97, s98
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s48, v20
+; SI-NEXT: s_lshr_b64 s[84:85], s[48:49], 16
; SI-NEXT: s_mov_b32 s87, s84
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_readfirstlane_b32 s6, v28
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s4, v36
+; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: s_lshr_b32 s7, s4, 16
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: v_writelane_b32 v61, s4, 30
+; SI-NEXT: v_writelane_b32 v61, s5, 31
+; SI-NEXT: v_readfirstlane_b32 s4, v39
+; SI-NEXT: s_lshr_b32 s5, s4, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s4, v38
+; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
+; SI-NEXT: v_readfirstlane_b32 s5, v8
+; SI-NEXT: s_lshr_b64 s[90:91], s[6:7], 16
+; SI-NEXT: s_lshr_b32 s7, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v37
+; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 16
+; SI-NEXT: s_mov_b32 s6, s92
+; SI-NEXT: v_writelane_b32 v62, s6, 36
+; SI-NEXT: v_writelane_b32 v62, s7, 37
+; SI-NEXT: s_mov_b32 s6, s60
+; SI-NEXT: v_writelane_b32 v62, s6, 56
+; SI-NEXT: s_mov_b32 s5, s8
+; SI-NEXT: v_writelane_b32 v62, s7, 57
+; SI-NEXT: s_mov_b32 s14, s8
+; SI-NEXT: s_lshr_b32 s6, s8, 8
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 4
+; SI-NEXT: v_writelane_b32 v62, s9, 5
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 2
+; SI-NEXT: v_writelane_b32 v62, s9, 3
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 0
+; SI-NEXT: v_writelane_b32 v62, s9, 1
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 10
+; SI-NEXT: v_writelane_b32 v62, s9, 11
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 16
; SI-NEXT: v_readfirstlane_b32 s68, v26
+; SI-NEXT: v_writelane_b32 v62, s8, 8
; SI-NEXT: s_lshr_b64 s[70:71], s[68:69], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 9
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 8
; SI-NEXT: s_mov_b32 s81, s70
-; SI-NEXT: v_readfirstlane_b32 s92, v30
-; SI-NEXT: s_lshr_b64 s[48:49], s[92:93], 16
-; SI-NEXT: s_mov_b32 s67, s48
+; SI-NEXT: v_writelane_b32 v62, s8, 6
+; SI-NEXT: v_writelane_b32 v62, s9, 7
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 16
+; SI-NEXT: v_writelane_b32 v62, s9, 17
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 16
+; SI-NEXT: v_readfirstlane_b32 s74, v30
+; SI-NEXT: v_writelane_b32 v62, s8, 14
+; SI-NEXT: s_lshr_b64 s[64:65], s[74:75], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 15
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 8
+; SI-NEXT: s_mov_b32 s67, s64
+; SI-NEXT: v_writelane_b32 v62, s8, 12
+; SI-NEXT: v_writelane_b32 v62, s9, 13
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 22
+; SI-NEXT: v_writelane_b32 v62, s9, 23
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 16
; SI-NEXT: v_readfirstlane_b32 s38, v33
-; SI-NEXT: s_lshr_b64 s[98:99], s[38:39], 16
-; SI-NEXT: s_mov_b32 s53, s98
-; SI-NEXT: s_mov_b32 s35, s82
-; SI-NEXT: v_readfirstlane_b32 s56, v49
-; SI-NEXT: s_lshr_b64 s[30:31], s[56:57], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 20
+; SI-NEXT: s_lshr_b64 s[50:51], s[38:39], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 21
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 8
+; SI-NEXT: s_mov_b32 s53, s50
+; SI-NEXT: v_writelane_b32 v62, s8, 18
+; SI-NEXT: v_writelane_b32 v62, s9, 19
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 28
+; SI-NEXT: v_writelane_b32 v62, s9, 29
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 26
+; SI-NEXT: v_writelane_b32 v62, s9, 27
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 8
+; SI-NEXT: s_mov_b32 s35, s92
+; SI-NEXT: v_writelane_b32 v62, s8, 24
+; SI-NEXT: v_writelane_b32 v62, s9, 25
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 34
+; SI-NEXT: v_writelane_b32 v62, s9, 35
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 16
+; SI-NEXT: v_readfirstlane_b32 s42, v45
+; SI-NEXT: v_writelane_b32 v62, s8, 32
+; SI-NEXT: s_lshr_b64 s[30:31], s[42:43], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 33
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 8
; SI-NEXT: s_mov_b32 s79, s30
-; SI-NEXT: v_readfirstlane_b32 s88, v2
-; SI-NEXT: s_lshr_b64 s[74:75], s[88:89], 16
-; SI-NEXT: s_mov_b32 s95, s74
-; SI-NEXT: v_mov_b32_e32 v38, v16
-; SI-NEXT: v_readfirstlane_b32 s72, v38
-; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 30
+; SI-NEXT: v_writelane_b32 v62, s9, 31
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 42
+; SI-NEXT: v_writelane_b32 v62, s9, 43
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 16
+; SI-NEXT: v_readfirstlane_b32 s88, v52
+; SI-NEXT: v_writelane_b32 v62, s8, 40
+; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 41
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 8
+; SI-NEXT: s_mov_b32 s95, s82
+; SI-NEXT: v_writelane_b32 v62, s8, 38
+; SI-NEXT: v_writelane_b32 v62, s9, 39
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 48
+; SI-NEXT: v_writelane_b32 v62, s9, 49
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 46
+; SI-NEXT: v_writelane_b32 v62, s9, 47
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 8
; SI-NEXT: s_mov_b32 s77, s60
-; SI-NEXT: v_readfirstlane_b32 s58, v44
-; SI-NEXT: s_lshr_b64 s[54:55], s[58:59], 16
-; SI-NEXT: s_mov_b32 s63, s54
-; SI-NEXT: v_mov_b32_e32 v15, v17
+; SI-NEXT: v_writelane_b32 v62, s8, 44
+; SI-NEXT: v_writelane_b32 v62, s9, 45
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 54
+; SI-NEXT: v_mov_b32_e32 v49, v32
+; SI-NEXT: v_writelane_b32 v62, s9, 55
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 16
+; SI-NEXT: v_readfirstlane_b32 s58, v49
+; SI-NEXT: v_writelane_b32 v62, s8, 52
+; SI-NEXT: s_lshr_b64 s[56:57], s[58:59], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 53
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 8
+; SI-NEXT: s_mov_b32 s63, s56
+; SI-NEXT: v_writelane_b32 v62, s8, 50
+; SI-NEXT: v_writelane_b32 v62, s9, 51
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 62
+; SI-NEXT: v_writelane_b32 v62, s9, 63
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 16
; SI-NEXT: v_readfirstlane_b32 s44, v15
-; SI-NEXT: s_lshr_b64 s[42:43], s[44:45], 16
-; SI-NEXT: s_mov_b32 s47, s42
-; SI-NEXT: v_readfirstlane_b32 s28, v3
+; SI-NEXT: v_writelane_b32 v62, s8, 60
+; SI-NEXT: s_lshr_b64 s[54:55], s[44:45], 16
+; SI-NEXT: v_writelane_b32 v62, s9, 61
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 8
+; SI-NEXT: s_mov_b32 s47, s54
+; SI-NEXT: v_writelane_b32 v62, s8, 58
+; SI-NEXT: v_writelane_b32 v62, s9, 59
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 4
+; SI-NEXT: v_mov_b32_e32 v32, v43
+; SI-NEXT: v_writelane_b32 v61, s9, 5
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 16
+; SI-NEXT: v_readfirstlane_b32 s28, v32
+; SI-NEXT: v_writelane_b32 v61, s8, 2
; SI-NEXT: s_lshr_b64 s[26:27], s[28:29], 16
+; SI-NEXT: v_writelane_b32 v61, s9, 3
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 8
; SI-NEXT: s_mov_b32 s41, s26
-; SI-NEXT: v_mov_b32_e32 v17, v28
-; SI-NEXT: v_readfirstlane_b32 s22, v17
-; SI-NEXT: s_lshr_b64 s[36:37], s[22:23], 16
-; SI-NEXT: s_mov_b32 s25, s36
-; SI-NEXT: v_mov_b32_e32 v28, v55
-; SI-NEXT: v_readfirstlane_b32 s18, v28
-; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
-; SI-NEXT: s_mov_b32 s17, s14
+; SI-NEXT: v_writelane_b32 v61, s8, 0
+; SI-NEXT: v_writelane_b32 v61, s9, 1
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 10
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_mov_b32_e32 v1, v9
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: v_mov_b32_e32 v6, v55
+; SI-NEXT: v_writelane_b32 v61, s9, 11
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
+; SI-NEXT: v_readfirstlane_b32 s22, v6
+; SI-NEXT: v_writelane_b32 v61, s8, 8
+; SI-NEXT: s_lshr_b64 s[20:21], s[22:23], 16
+; SI-NEXT: v_writelane_b32 v61, s9, 9
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 8
+; SI-NEXT: s_mov_b32 s25, s20
+; SI-NEXT: v_writelane_b32 v61, s8, 6
+; SI-NEXT: v_writelane_b32 v61, s9, 7
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 16
+; SI-NEXT: v_writelane_b32 v61, s9, 17
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
+; SI-NEXT: v_readfirstlane_b32 s18, v11
+; SI-NEXT: v_writelane_b32 v61, s8, 14
+; SI-NEXT: s_lshr_b64 s[36:37], s[18:19], 16
+; SI-NEXT: v_writelane_b32 v61, s9, 15
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 8
+; SI-NEXT: s_mov_b32 s17, s36
+; SI-NEXT: v_writelane_b32 v61, s8, 12
+; SI-NEXT: v_writelane_b32 v61, s9, 13
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 22
+; SI-NEXT: v_writelane_b32 v61, s9, 23
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 20
+; SI-NEXT: v_writelane_b32 v61, s9, 21
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
+; SI-NEXT: s_mov_b32 s11, s90
+; SI-NEXT: v_writelane_b32 v61, s8, 18
+; SI-NEXT: v_writelane_b32 v61, s9, 19
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 28
+; SI-NEXT: v_writelane_b32 v61, s9, 29
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 26
+; SI-NEXT: v_writelane_b32 v61, s9, 27
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 24
+; SI-NEXT: v_writelane_b32 v61, s9, 25
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 36
+; SI-NEXT: v_writelane_b32 v61, s9, 37
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 34
+; SI-NEXT: v_writelane_b32 v61, s9, 35
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 32
; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v7
; SI-NEXT: v_lshrrev_b32_e32 v24, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v20
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v53
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v21
+; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v53
; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v8
-; SI-NEXT: s_lshr_b32 s88, s96, 8
+; SI-NEXT: s_lshr_b32 s88, s98, 8
; SI-NEXT: s_lshr_b32 s61, s84, 8
; SI-NEXT: s_lshr_b32 s72, s70, 8
-; SI-NEXT: s_lshr_b32 s75, s48, 8
-; SI-NEXT: s_lshr_b32 s58, s98, 8
-; SI-NEXT: s_lshr_b32 s43, s82, 8
+; SI-NEXT: s_lshr_b32 s57, s64, 8
+; SI-NEXT: s_lshr_b32 s58, s50, 8
+; SI-NEXT: s_lshr_b32 s93, s92, 8
; SI-NEXT: s_lshr_b32 s44, s30, 8
-; SI-NEXT: s_mov_b32 s64, s74
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s4, v13
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s12, v48
-; SI-NEXT: s_lshr_b32 s27, s74, 8
-; SI-NEXT: s_mov_b32 s92, s60
+; SI-NEXT: s_lshr_b32 s27, s82, 8
; SI-NEXT: s_lshr_b32 s28, s60, 8
-; SI-NEXT: s_lshr_b32 s74, s54, 8
-; SI-NEXT: s_mov_b32 s68, s42
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s4, v36
-; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: s_lshr_b32 s13, s4, 16
-; SI-NEXT: s_mov_b32 s5, s13
-; SI-NEXT: v_writelane_b32 v61, s4, 26
-; SI-NEXT: v_writelane_b32 v61, s5, 27
-; SI-NEXT: v_readfirstlane_b32 s4, v39
-; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v51
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s5, v8
-; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
-; SI-NEXT: s_lshr_b32 s13, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s12, v37
-; SI-NEXT: s_lshr_b64 vcc, s[12:13], 16
-; SI-NEXT: s_mov_b32 s5, vcc_lo
-; SI-NEXT: s_mov_b32 s90, vcc_lo
-; SI-NEXT: s_lshr_b32 s6, vcc_lo, 8
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 4
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 5
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 2
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 3
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 0
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 1
-; SI-NEXT: s_lshr_b64 vcc, s[86:87], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 10
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 11
-; SI-NEXT: s_lshr_b64 vcc, s[86:87], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 8
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 9
-; SI-NEXT: s_lshr_b64 vcc, s[86:87], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 6
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 7
-; SI-NEXT: s_lshr_b64 vcc, s[80:81], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 16
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 17
-; SI-NEXT: s_lshr_b64 vcc, s[80:81], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 14
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 15
-; SI-NEXT: s_lshr_b64 vcc, s[80:81], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 12
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 13
-; SI-NEXT: s_lshr_b64 vcc, s[66:67], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 22
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 23
-; SI-NEXT: s_lshr_b64 vcc, s[66:67], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 20
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 21
-; SI-NEXT: s_lshr_b64 vcc, s[66:67], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 18
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 19
-; SI-NEXT: s_lshr_b64 vcc, s[52:53], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 28
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 29
-; SI-NEXT: s_lshr_b64 vcc, s[52:53], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 26
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 27
-; SI-NEXT: s_lshr_b64 vcc, s[52:53], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 24
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 25
-; SI-NEXT: s_lshr_b64 vcc, s[34:35], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 34
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 35
-; SI-NEXT: s_lshr_b64 vcc, s[34:35], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 32
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 33
-; SI-NEXT: s_lshr_b64 vcc, s[34:35], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 30
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 31
-; SI-NEXT: s_lshr_b64 vcc, s[78:79], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 40
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 41
-; SI-NEXT: s_lshr_b64 vcc, s[78:79], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 38
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 39
-; SI-NEXT: s_lshr_b64 vcc, s[78:79], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 36
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 37
-; SI-NEXT: s_lshr_b64 vcc, s[94:95], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 46
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 47
-; SI-NEXT: s_lshr_b64 vcc, s[94:95], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 44
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 45
-; SI-NEXT: s_lshr_b64 vcc, s[94:95], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 42
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 43
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 52
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 53
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 50
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 51
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 48
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 49
-; SI-NEXT: s_lshr_b64 vcc, s[62:63], 24
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 58
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 59
-; SI-NEXT: s_lshr_b64 vcc, s[62:63], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 56
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 57
-; SI-NEXT: s_lshr_b64 vcc, s[62:63], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 54
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 55
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 0
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 1
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 62
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 63
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 8
-; SI-NEXT: v_writelane_b32 v62, vcc_lo, 60
-; SI-NEXT: v_writelane_b32 v62, vcc_hi, 61
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 6
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 7
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 16
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 4
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 5
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 8
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 2
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 3
-; SI-NEXT: s_lshr_b64 vcc, s[24:25], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 12
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 13
-; SI-NEXT: s_lshr_b64 vcc, s[24:25], 16
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 10
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 11
-; SI-NEXT: s_lshr_b64 vcc, s[24:25], 8
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 8
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 9
-; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 18
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 19
-; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 16
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 17
-; SI-NEXT: s_lshr_b64 vcc, s[16:17], 8
-; SI-NEXT: s_mov_b32 s11, s20
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 14
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 15
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 24
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 25
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 16
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 22
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 23
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 8
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 20
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 21
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 24
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 32
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 33
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 30
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 31
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 8
-; SI-NEXT: v_writelane_b32 v61, vcc_lo, 28
-; SI-NEXT: s_lshr_b32 s22, s42, 8
-; SI-NEXT: s_mov_b32 s56, s26
-; SI-NEXT: s_lshr_b32 s21, s26, 8
-; SI-NEXT: s_lshr_b32 s18, s36, 8
-; SI-NEXT: s_mov_b32 s38, s14
-; SI-NEXT: s_lshr_b32 s15, s14, 8
-; SI-NEXT: s_mov_b32 s14, s20
-; SI-NEXT: s_lshr_b32 s12, s20, 8
-; SI-NEXT: v_writelane_b32 v61, vcc_hi, 29
-; SI-NEXT: s_mov_b64 vcc, 0
-; SI-NEXT: v_lshrrev_b32_e32 v19, 24, v10
+; SI-NEXT: s_mov_b32 s74, s56
+; SI-NEXT: s_lshr_b32 s21, s56, 8
+; SI-NEXT: s_lshr_b32 s22, s54, 8
+; SI-NEXT: s_mov_b32 s68, s26
+; SI-NEXT: s_lshr_b32 s56, s26, 8
+; SI-NEXT: s_mov_b32 s42, s20
+; SI-NEXT: s_lshr_b32 s18, s20, 8
+; SI-NEXT: s_lshr_b32 s91, s36, 8
+; SI-NEXT: s_mov_b32 s38, s90
+; SI-NEXT: s_lshr_b32 s12, s90, 8
+; SI-NEXT: v_writelane_b32 v61, s9, 33
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: v_lshrrev_b32_e32 v20, 24, v10
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_mov_b32_e32 v16, v14
; SI-NEXT: v_lshrrev_b32_e32 v43, 24, v14
; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v56
-; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v21
-; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v22
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v48
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v34, 24, v41
; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v46
; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v58
@@ -157613,57 +158527,50 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v18
; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v20, v35
-; SI-NEXT: v_mov_b32_e32 v14, v52
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v37, v45
-; SI-NEXT: v_mov_b32_e32 v45, v2
+; SI-NEXT: v_mov_b32_e32 v19, v35
; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v39, v4
-; SI-NEXT: v_mov_b32_e32 v52, v44
-; SI-NEXT: v_mov_b32_e32 v44, v11
-; SI-NEXT: v_mov_b32_e32 v35, v3
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, v17
-; SI-NEXT: v_mov_b32_e32 v11, v6
+; SI-NEXT: v_mov_b32_e32 v39, v45
+; SI-NEXT: v_mov_b32_e32 v45, v1
+; SI-NEXT: v_mov_b32_e32 v14, v51
+; SI-NEXT: v_mov_b32_e32 v37, v4
+; SI-NEXT: v_mov_b32_e32 v51, v40
+; SI-NEXT: v_mov_b32_e32 v40, v42
+; SI-NEXT: v_mov_b32_e32 v42, v2
+; SI-NEXT: v_mov_b32_e32 v35, v32
+; SI-NEXT: v_mov_b32_e32 v1, v6
; SI-NEXT: v_mov_b32_e32 v6, v10
; SI-NEXT: v_mov_b32_e32 v10, v7
-; SI-NEXT: v_mov_b32_e32 v17, v28
-; SI-NEXT: v_mov_b32_e32 v16, v23
-; SI-NEXT: v_mov_b32_e32 v23, v32
-; SI-NEXT: v_mov_b32_e32 v28, v48
-; SI-NEXT: v_mov_b32_e32 v32, v13
-; SI-NEXT: v_mov_b32_e32 v48, v22
+; SI-NEXT: v_mov_b32_e32 v32, v17
; SI-NEXT: v_mov_b32_e32 v7, v5
-; SI-NEXT: v_mov_b32_e32 v2, v12
-; SI-NEXT: v_mov_b32_e32 v3, v18
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v3, v12
+; SI-NEXT: v_mov_b32_e32 v2, v18
; SI-NEXT: s_branch .LBB91_3
; SI-NEXT: .LBB91_2:
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v37, v40
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v19, v51
; SI-NEXT: v_writelane_b32 v62, s4, 0
; SI-NEXT: v_writelane_b32 v62, s5, 1
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v45, v11
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v39, v38
; SI-NEXT: v_writelane_b32 v62, s4, 2
; SI-NEXT: v_writelane_b32 v62, s5, 3
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v40, v51
+; SI-NEXT: v_mov_b32_e32 v51, v49
; SI-NEXT: v_writelane_b32 v62, s4, 4
; SI-NEXT: v_writelane_b32 v62, s5, 5
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v52, v32
+; SI-NEXT: v_mov_b32_e32 v60, v36
; SI-NEXT: v_writelane_b32 v62, s4, 6
; SI-NEXT: v_writelane_b32 v62, s5, 7
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v1, v28
+; SI-NEXT: v_mov_b32_e32 v49, v32
; SI-NEXT: v_writelane_b32 v62, s4, 8
; SI-NEXT: v_writelane_b32 v62, s5, 9
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v11, v36
+; SI-NEXT: v_mov_b32_e32 v15, v28
; SI-NEXT: v_writelane_b32 v62, s4, 10
; SI-NEXT: v_writelane_b32 v62, s5, 11
; SI-NEXT: ; implicit-def: $sgpr4
@@ -157673,130 +158580,126 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s5, 13
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 14
; SI-NEXT: v_writelane_b32 v62, s5, 15
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: v_mov_b32_e32 v9, v6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: v_mov_b32_e32 v16, v14
; SI-NEXT: v_writelane_b32 v62, s4, 16
; SI-NEXT: v_writelane_b32 v62, s5, 17
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_mov_b64 vcc, -1
+; SI-NEXT: v_mov_b32_e32 v9, v6
; SI-NEXT: v_writelane_b32 v62, s4, 18
; SI-NEXT: v_writelane_b32 v62, s5, 19
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v20, v49
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: v_writelane_b32 v62, s4, 20
; SI-NEXT: v_writelane_b32 v62, s5, 21
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v49, v2
+; SI-NEXT: v_mov_b32_e32 v14, v54
; SI-NEXT: v_writelane_b32 v62, s4, 22
; SI-NEXT: v_writelane_b32 v62, s5, 23
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v14, v54
+; SI-NEXT: v_mov_b32_e32 v37, v40
; SI-NEXT: v_writelane_b32 v62, s4, 24
; SI-NEXT: v_writelane_b32 v62, s5, 25
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v52, v13
; SI-NEXT: v_writelane_b32 v62, s4, 26
; SI-NEXT: v_writelane_b32 v62, s5, 27
; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $vgpr1
; SI-NEXT: v_mov_b32_e32 v54, v42
; SI-NEXT: v_writelane_b32 v62, s4, 28
; SI-NEXT: v_writelane_b32 v62, s5, 29
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v38, v16
+; SI-NEXT: v_mov_b32_e32 v40, v50
; SI-NEXT: v_writelane_b32 v62, s4, 30
; SI-NEXT: v_writelane_b32 v62, s5, 31
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v42, v50
+; SI-NEXT: v_mov_b32_e32 v42, v2
; SI-NEXT: v_writelane_b32 v62, s4, 32
; SI-NEXT: v_writelane_b32 v62, s5, 33
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v15, v17
+; SI-NEXT: v_mov_b32_e32 v35, v43
; SI-NEXT: v_writelane_b32 v62, s4, 34
; SI-NEXT: v_writelane_b32 v62, s5, 35
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v35, v23
+; SI-NEXT: v_mov_b32_e32 v1, v55
; SI-NEXT: v_writelane_b32 v62, s4, 36
; SI-NEXT: v_writelane_b32 v62, s5, 37
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v6, v21
+; SI-NEXT: v_mov_b32_e32 v13, v23
; SI-NEXT: v_writelane_b32 v62, s4, 38
; SI-NEXT: v_writelane_b32 v62, s5, 39
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v17, v55
+; SI-NEXT: v_mov_b32_e32 v6, v21
; SI-NEXT: v_writelane_b32 v62, s4, 40
; SI-NEXT: v_writelane_b32 v62, s5, 41
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v16, v43
+; SI-NEXT: v_mov_b32_e32 v23, v17
; SI-NEXT: v_writelane_b32 v62, s4, 42
; SI-NEXT: v_writelane_b32 v62, s5, 43
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v23, v13
+; SI-NEXT: v_mov_b32_e32 v21, v22
; SI-NEXT: v_writelane_b32 v62, s4, 44
; SI-NEXT: v_writelane_b32 v62, s5, 45
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v21, v22
+; SI-NEXT: v_mov_b32_e32 v7, v5
; SI-NEXT: v_writelane_b32 v62, s4, 46
; SI-NEXT: v_writelane_b32 v62, s5, 47
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v7, v5
+; SI-NEXT: v_mov_b32_e32 v3, v12
; SI-NEXT: v_writelane_b32 v62, s4, 48
; SI-NEXT: v_writelane_b32 v62, s5, 49
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v2, v12
+; SI-NEXT: v_mov_b32_e32 v2, v18
; SI-NEXT: v_writelane_b32 v62, s4, 50
; SI-NEXT: v_writelane_b32 v62, s5, 51
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v3, v18
-; SI-NEXT: v_writelane_b32 v62, s4, 52
-; SI-NEXT: v_writelane_b32 v62, s5, 53
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $sgpr86
; SI-NEXT: ; implicit-def: $sgpr84
; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr49
; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr20
; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr39
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr78
; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $sgpr27
; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr21
; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $sgpr22
; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $sgpr18
; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr91
; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $vgpr27
@@ -157804,35 +158707,36 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; kill: killed $vgpr26
; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr43
; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr82
; SI-NEXT: ; implicit-def: $sgpr89
; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr73
; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr54
; SI-NEXT: ; implicit-def: $sgpr45
; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr68
; SI-NEXT: ; implicit-def: $sgpr29
; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr42
; SI-NEXT: ; implicit-def: $sgpr23
; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr36
; SI-NEXT: ; implicit-def: $sgpr19
; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $vgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr7
; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: v_writelane_b32 v62, s4, 52
+; SI-NEXT: v_writelane_b32 v62, s5, 53
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: v_writelane_b32 v62, s4, 54
; SI-NEXT: v_writelane_b32 v62, s5, 55
; SI-NEXT: ; implicit-def: $sgpr4
@@ -157886,84 +158790,110 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: v_writelane_b32 v61, s4, 24
; SI-NEXT: v_writelane_b32 v61, s5, 25
-; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: v_writelane_b32 v61, s4, 26
; SI-NEXT: v_writelane_b32 v61, s5, 27
-; SI-NEXT: v_writelane_b32 v61, s20, 28
-; SI-NEXT: v_writelane_b32 v61, s21, 29
-; SI-NEXT: ; implicit-def: $sgpr20
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v61, s20, 30
-; SI-NEXT: v_writelane_b32 v61, s21, 31
-; SI-NEXT: v_writelane_b32 v61, s90, 32
-; SI-NEXT: v_writelane_b32 v61, s91, 33
-; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: v_writelane_b32 v61, s4, 28
+; SI-NEXT: v_writelane_b32 v61, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: v_writelane_b32 v61, s4, 30
+; SI-NEXT: v_writelane_b32 v61, s5, 31
+; SI-NEXT: v_writelane_b32 v61, s14, 32
+; SI-NEXT: v_writelane_b32 v61, s15, 33
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v61, s14, 34
+; SI-NEXT: v_writelane_b32 v61, s15, 35
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_writelane_b32 v61, s14, 36
+; SI-NEXT: v_writelane_b32 v61, s15, 37
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB91_3: ; %Flow
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, vcc
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v26, v27
; SI-NEXT: v_mov_b32_e32 v27, v29
; SI-NEXT: v_mov_b32_e32 v29, v30
; SI-NEXT: v_mov_b32_e32 v30, v31
; SI-NEXT: v_mov_b32_e32 v31, v33
-; SI-NEXT: s_cbranch_vccnz .LBB91_5
+; SI-NEXT: s_cbranch_scc1 .LBB91_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v51
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_readfirstlane_b32 s4, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v18
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v32
+; SI-NEXT: s_lshr_b32 s7, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v4
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s12, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v36
+; SI-NEXT: s_lshr_b64 s[90:91], s[6:7], 16
; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v36
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v10
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_readfirstlane_b32 s8, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v28
; SI-NEXT: v_readfirstlane_b32 s6, v12
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: s_lshr_b32 s13, s6, 16
; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v28
-; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s12, v4
+; SI-NEXT: s_mov_b32 s9, s13
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v23
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; SI-NEXT: v_writelane_b32 v61, s8, 30
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: s_mov_b32 s7, s9
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_writelane_b32 v61, s9, 31
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_writelane_b32 v61, s6, 26
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; SI-NEXT: v_writelane_b32 v61, s7, 27
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_readfirstlane_b32 s18, v4
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v51
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_mov_b32_e32 v17, v16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
+; SI-NEXT: s_mov_b32 s11, s20
+; SI-NEXT: s_mov_b32 s5, s90
+; SI-NEXT: v_lshrrev_b32_e32 v20, 24, v11
+; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v12
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
-; SI-NEXT: s_lshr_b64 s[20:21], s[8:9], 16
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v17
; SI-NEXT: v_readfirstlane_b32 s6, v5
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s19, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s18, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s8, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
@@ -157980,41 +158910,26 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_lshr_b64 s[36:37], s[22:23], 16
-; SI-NEXT: s_mov_b32 s25, s36
-; SI-NEXT: s_lshr_b64 s[26:27], s[18:19], 16
-; SI-NEXT: s_mov_b32 s17, s26
-; SI-NEXT: s_mov_b32 s11, s20
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s5, v18
-; SI-NEXT: s_lshr_b32 s13, s5, 16
-; SI-NEXT: s_lshr_b64 vcc, s[12:13], 16
-; SI-NEXT: s_mov_b32 s5, vcc_lo
-; SI-NEXT: s_lshr_b32 s18, s36, 8
-; SI-NEXT: s_lshr_b32 s12, s20, 8
-; SI-NEXT: v_lshrrev_b32_e32 v19, 24, v11
+; SI-NEXT: s_lshr_b64 s[26:27], s[22:23], 16
+; SI-NEXT: s_mov_b32 s25, s26
+; SI-NEXT: s_lshr_b64 s[36:37], s[18:19], 16
+; SI-NEXT: s_mov_b32 s17, s36
+; SI-NEXT: s_lshr_b32 s18, s26, 8
+; SI-NEXT: s_lshr_b32 s91, s36, 8
; SI-NEXT: v_lshrrev_b32_e32 v29, 24, v2
-; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v12
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_readfirstlane_b32 s6, v4
+; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_readfirstlane_b32 s8, v3
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_readfirstlane_b32 s6, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; SI-NEXT: v_readfirstlane_b32 s28, v4
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v59
-; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v58
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
@@ -158033,7 +158948,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v52
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v49
; SI-NEXT: v_readfirstlane_b32 s6, v7
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s45, s6, 16
@@ -158046,7 +158961,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v38
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v60
; SI-NEXT: v_readfirstlane_b32 s6, v8
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s59, s6, 16
@@ -158059,7 +158974,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v52
; SI-NEXT: v_readfirstlane_b32 s6, v9
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s73, s6, 16
@@ -158073,15 +158988,15 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v10
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v14
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v39
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s89, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v10
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v21
-; SI-NEXT: v_readfirstlane_b32 s56, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v20
+; SI-NEXT: v_readfirstlane_b32 s42, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v19
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v10
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
@@ -158090,29 +159005,29 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s8, v4
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: s_lshr_b32 s57, s6, 16
+; SI-NEXT: s_lshr_b32 s43, s6, 16
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[30:31], s[56:57], 16
-; SI-NEXT: s_mov_b32 s79, s30
-; SI-NEXT: s_lshr_b64 s[50:51], s[88:89], 16
-; SI-NEXT: s_mov_b32 s95, s50
-; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
-; SI-NEXT: s_mov_b32 s77, s74
-; SI-NEXT: s_lshr_b64 s[54:55], s[58:59], 16
-; SI-NEXT: s_mov_b32 s63, s54
-; SI-NEXT: s_lshr_b64 s[60:61], s[44:45], 16
-; SI-NEXT: s_mov_b32 s47, s60
-; SI-NEXT: s_lshr_b64 s[42:43], s[28:29], 16
-; SI-NEXT: s_mov_b32 s41, s42
-; SI-NEXT: s_lshr_b32 s44, s30, 8
-; SI-NEXT: s_lshr_b32 s27, s50, 8
-; SI-NEXT: s_lshr_b32 s28, s74, 8
-; SI-NEXT: s_lshr_b32 s22, s60, 8
-; SI-NEXT: s_mov_b32 s56, s42
-; SI-NEXT: s_lshr_b32 s21, s42, 8
-; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v14
-; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v13
+; SI-NEXT: s_lshr_b64 s[12:13], s[42:43], 16
+; SI-NEXT: s_mov_b32 s79, s12
+; SI-NEXT: s_lshr_b64 s[82:83], s[88:89], 16
+; SI-NEXT: s_mov_b32 s95, s82
+; SI-NEXT: s_lshr_b64 s[92:93], s[72:73], 16
+; SI-NEXT: s_mov_b32 s77, s92
+; SI-NEXT: s_lshr_b64 s[60:61], s[58:59], 16
+; SI-NEXT: s_mov_b32 s63, s60
+; SI-NEXT: s_lshr_b64 s[54:55], s[44:45], 16
+; SI-NEXT: s_mov_b32 s47, s54
+; SI-NEXT: s_lshr_b64 s[56:57], s[28:29], 16
+; SI-NEXT: s_mov_b32 s41, s56
+; SI-NEXT: s_lshr_b32 s44, s12, 8
+; SI-NEXT: s_lshr_b32 s27, s82, 8
+; SI-NEXT: s_lshr_b32 s28, s92, 8
+; SI-NEXT: s_lshr_b32 s21, s60, 8
+; SI-NEXT: s_lshr_b32 s22, s54, 8
+; SI-NEXT: s_mov_b32 s42, s26
+; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v14
+; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v13
; SI-NEXT: v_lshrrev_b32_e32 v34, 24, v8
; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v7
; SI-NEXT: v_lshrrev_b32_e32 v30, 24, v3
@@ -158123,18 +159038,18 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s90, v10
+; SI-NEXT: v_readfirstlane_b32 s14, v10
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[34:35], s[8:9], 16
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v56
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_readfirstlane_b32 s6, v4
-; SI-NEXT: s_lshr_b32 s91, s6, 16
-; SI-NEXT: s_lshr_b64 s[82:83], s[90:91], 16
-; SI-NEXT: s_mov_b32 s35, s82
-; SI-NEXT: s_lshr_b32 s43, s82, 8
-; SI-NEXT: s_mov_b32 s90, vcc_lo
+; SI-NEXT: s_lshr_b32 s15, s6, 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[14:15], 16
+; SI-NEXT: s_mov_b32 s35, s30
+; SI-NEXT: s_lshr_b32 s93, s30, 8
+; SI-NEXT: s_mov_b32 s14, s90
; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v4
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
@@ -158155,12 +159070,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_readfirstlane_b32 s6, v10
; SI-NEXT: s_lshr_b32 s39, s6, 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_lshr_b64 s[98:99], s[38:39], 16
-; SI-NEXT: s_mov_b32 s53, s98
-; SI-NEXT: s_lshr_b32 s58, s98, 8
-; SI-NEXT: s_mov_b32 s38, s26
+; SI-NEXT: s_lshr_b64 s[50:51], s[38:39], 16
+; SI-NEXT: s_mov_b32 s53, s50
+; SI-NEXT: s_lshr_b32 s58, s50, 8
+; SI-NEXT: s_mov_b32 s38, s20
; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v10
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
@@ -158171,33 +159086,36 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s6, v16
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[66:67], s[8:9], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v24, 24, v6
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v15
-; SI-NEXT: v_readfirstlane_b32 s92, v16
+; SI-NEXT: v_readfirstlane_b32 s74, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_readfirstlane_b32 s6, v15
-; SI-NEXT: s_lshr_b32 s93, s6, 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[92:93], 16
-; SI-NEXT: s_mov_b32 s67, s48
-; SI-NEXT: s_lshr_b32 s75, s48, 8
-; SI-NEXT: s_mov_b32 s92, s74
-; SI-NEXT: s_lshr_b32 s74, s54, 8
+; SI-NEXT: s_lshr_b32 s75, s6, 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[74:75], 16
+; SI-NEXT: s_mov_b32 s67, s64
+; SI-NEXT: s_lshr_b32 s57, s64, 8
+; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: v_lshrrev_b32_e32 v43, 24, v15
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v1
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s8, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v24, 24, v6
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s68, v16
@@ -158213,11 +159131,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b64 s[70:71], s[68:69], 16
; SI-NEXT: s_mov_b32 s81, s70
; SI-NEXT: s_lshr_b32 s72, s70, 8
-; SI-NEXT: s_mov_b32 s68, s60
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v1
+; SI-NEXT: s_mov_b32 s68, s56
+; SI-NEXT: s_lshr_b32 s56, s56, 8
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
@@ -158226,7 +159141,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s64, v16
+; SI-NEXT: v_readfirstlane_b32 s48, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -158235,11 +159150,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[86:87], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v6
-; SI-NEXT: s_lshr_b32 s65, s6, 16
-; SI-NEXT: s_lshr_b64 s[84:85], s[64:65], 16
+; SI-NEXT: s_lshr_b32 s49, s6, 16
+; SI-NEXT: s_lshr_b64 s[84:85], s[48:49], 16
; SI-NEXT: s_mov_b32 s87, s84
; SI-NEXT: s_lshr_b32 s61, s84, 8
-; SI-NEXT: s_mov_b32 s64, s50
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
@@ -158251,9 +159165,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s6, v17
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v1
-; SI-NEXT: s_lshr_b32 s7, s6, 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s6, 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[8:9], 16
+; SI-NEXT: s_mov_b32 s6, s30
; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v9
+; SI-NEXT: s_mov_b32 s30, s12
+; SI-NEXT: s_lshr_b32 s12, s20, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v18
@@ -158261,178 +159178,181 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s6, v16
-; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
-; SI-NEXT: s_mov_b32 s9, s96
-; SI-NEXT: s_lshr_b64 s[14:15], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 4
-; SI-NEXT: v_writelane_b32 v62, s15, 5
-; SI-NEXT: s_lshr_b64 s[14:15], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 2
-; SI-NEXT: v_writelane_b32 v62, s15, 3
-; SI-NEXT: s_lshr_b64 s[14:15], s[8:9], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 0
-; SI-NEXT: v_writelane_b32 v62, s15, 1
-; SI-NEXT: s_lshr_b64 s[14:15], s[86:87], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 10
-; SI-NEXT: v_writelane_b32 v62, s15, 11
-; SI-NEXT: s_lshr_b64 s[14:15], s[86:87], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 8
-; SI-NEXT: v_writelane_b32 v62, s15, 9
-; SI-NEXT: s_lshr_b64 s[14:15], s[86:87], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 6
-; SI-NEXT: v_writelane_b32 v62, s15, 7
-; SI-NEXT: s_lshr_b64 s[14:15], s[80:81], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 16
-; SI-NEXT: v_writelane_b32 v62, s15, 17
-; SI-NEXT: s_lshr_b64 s[14:15], s[80:81], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 14
-; SI-NEXT: v_writelane_b32 v62, s15, 15
-; SI-NEXT: s_lshr_b64 s[14:15], s[80:81], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 12
-; SI-NEXT: v_writelane_b32 v62, s15, 13
-; SI-NEXT: s_lshr_b64 s[14:15], s[66:67], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 22
-; SI-NEXT: v_writelane_b32 v62, s15, 23
-; SI-NEXT: s_lshr_b64 s[14:15], s[66:67], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 20
-; SI-NEXT: v_writelane_b32 v62, s15, 21
-; SI-NEXT: s_lshr_b64 s[14:15], s[66:67], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 18
-; SI-NEXT: v_writelane_b32 v62, s15, 19
-; SI-NEXT: s_lshr_b64 s[14:15], s[52:53], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 28
-; SI-NEXT: v_writelane_b32 v62, s15, 29
-; SI-NEXT: s_lshr_b64 s[14:15], s[52:53], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 26
-; SI-NEXT: v_writelane_b32 v62, s15, 27
-; SI-NEXT: s_lshr_b64 s[14:15], s[52:53], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 24
-; SI-NEXT: v_writelane_b32 v62, s15, 25
-; SI-NEXT: s_lshr_b64 s[14:15], s[34:35], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 34
-; SI-NEXT: v_writelane_b32 v62, s15, 35
-; SI-NEXT: s_lshr_b64 s[14:15], s[34:35], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 32
-; SI-NEXT: v_writelane_b32 v62, s15, 33
-; SI-NEXT: s_lshr_b64 s[14:15], s[34:35], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 30
-; SI-NEXT: v_writelane_b32 v62, s15, 31
-; SI-NEXT: s_lshr_b64 s[14:15], s[78:79], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 40
-; SI-NEXT: v_writelane_b32 v62, s15, 41
-; SI-NEXT: s_lshr_b64 s[14:15], s[78:79], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 38
-; SI-NEXT: v_writelane_b32 v62, s15, 39
-; SI-NEXT: s_lshr_b64 s[14:15], s[78:79], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 36
-; SI-NEXT: v_writelane_b32 v62, s15, 37
-; SI-NEXT: s_lshr_b64 s[14:15], s[94:95], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 46
-; SI-NEXT: v_writelane_b32 v62, s15, 47
-; SI-NEXT: s_lshr_b64 s[14:15], s[94:95], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 44
-; SI-NEXT: v_writelane_b32 v62, s15, 45
-; SI-NEXT: s_lshr_b64 s[14:15], s[94:95], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 42
-; SI-NEXT: v_writelane_b32 v62, s15, 43
-; SI-NEXT: s_lshr_b64 s[14:15], s[76:77], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 52
-; SI-NEXT: v_writelane_b32 v62, s15, 53
-; SI-NEXT: s_lshr_b64 s[14:15], s[76:77], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 50
-; SI-NEXT: v_writelane_b32 v62, s15, 51
-; SI-NEXT: s_lshr_b64 s[14:15], s[76:77], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 48
-; SI-NEXT: v_writelane_b32 v62, s15, 49
-; SI-NEXT: s_lshr_b64 s[14:15], s[62:63], 24
-; SI-NEXT: v_writelane_b32 v62, s14, 58
-; SI-NEXT: v_writelane_b32 v62, s15, 59
-; SI-NEXT: s_lshr_b64 s[14:15], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 56
-; SI-NEXT: v_writelane_b32 v62, s15, 57
-; SI-NEXT: s_lshr_b64 s[14:15], s[62:63], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 54
-; SI-NEXT: v_writelane_b32 v62, s15, 55
-; SI-NEXT: s_lshr_b64 s[14:15], s[46:47], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 0
-; SI-NEXT: v_writelane_b32 v61, s15, 1
-; SI-NEXT: s_lshr_b64 s[14:15], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v62, s14, 62
-; SI-NEXT: v_writelane_b32 v62, s15, 63
-; SI-NEXT: s_lshr_b64 s[14:15], s[46:47], 8
-; SI-NEXT: v_writelane_b32 v62, s14, 60
-; SI-NEXT: v_writelane_b32 v62, s15, 61
-; SI-NEXT: s_lshr_b64 s[14:15], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 6
-; SI-NEXT: v_writelane_b32 v61, s15, 7
-; SI-NEXT: s_lshr_b64 s[14:15], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v61, s14, 4
-; SI-NEXT: v_writelane_b32 v61, s15, 5
-; SI-NEXT: s_lshr_b64 s[14:15], s[40:41], 8
-; SI-NEXT: v_writelane_b32 v61, s14, 2
-; SI-NEXT: v_writelane_b32 v61, s15, 3
-; SI-NEXT: s_lshr_b64 s[14:15], s[24:25], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 12
-; SI-NEXT: v_writelane_b32 v61, s15, 13
-; SI-NEXT: s_lshr_b64 s[14:15], s[24:25], 16
-; SI-NEXT: v_writelane_b32 v61, s14, 10
-; SI-NEXT: v_writelane_b32 v61, s15, 11
-; SI-NEXT: s_lshr_b64 s[14:15], s[24:25], 8
-; SI-NEXT: v_writelane_b32 v61, s14, 8
-; SI-NEXT: v_writelane_b32 v61, s15, 9
-; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 18
-; SI-NEXT: v_writelane_b32 v61, s15, 19
-; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: v_writelane_b32 v61, s14, 16
-; SI-NEXT: v_writelane_b32 v61, s15, 17
-; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 8
-; SI-NEXT: v_writelane_b32 v61, s14, 14
-; SI-NEXT: v_writelane_b32 v61, s15, 15
-; SI-NEXT: s_lshr_b64 s[14:15], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 24
-; SI-NEXT: v_writelane_b32 v61, s15, 25
-; SI-NEXT: s_lshr_b64 s[14:15], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v61, s14, 22
-; SI-NEXT: v_writelane_b32 v61, s15, 23
-; SI-NEXT: s_lshr_b64 s[14:15], s[10:11], 8
-; SI-NEXT: v_writelane_b32 v61, s14, 20
-; SI-NEXT: v_writelane_b32 v61, s15, 21
-; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v61, s14, 32
-; SI-NEXT: v_writelane_b32 v61, s15, 33
-; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v61, s14, 30
-; SI-NEXT: v_writelane_b32 v61, s15, 31
-; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v61, s14, 28
-; SI-NEXT: v_writelane_b32 v61, s15, 29
-; SI-NEXT: s_lshr_b32 s88, s96, 8
-; SI-NEXT: s_lshr_b32 s15, s26, 8
-; SI-NEXT: s_mov_b32 s14, s20
-; SI-NEXT: s_lshr_b32 s6, vcc_lo, 8
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v16
+; SI-NEXT: s_lshr_b64 s[98:99], vcc, 16
+; SI-NEXT: s_mov_b32 s97, s98
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 4
+; SI-NEXT: v_writelane_b32 v62, s9, 5
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 2
+; SI-NEXT: v_writelane_b32 v62, s9, 3
+; SI-NEXT: s_lshr_b64 s[8:9], s[96:97], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 0
+; SI-NEXT: v_writelane_b32 v62, s9, 1
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 10
+; SI-NEXT: v_writelane_b32 v62, s9, 11
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 8
+; SI-NEXT: v_writelane_b32 v62, s9, 9
+; SI-NEXT: s_lshr_b64 s[8:9], s[86:87], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 6
+; SI-NEXT: v_writelane_b32 v62, s9, 7
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 16
+; SI-NEXT: v_writelane_b32 v62, s9, 17
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 14
+; SI-NEXT: v_writelane_b32 v62, s9, 15
+; SI-NEXT: s_lshr_b64 s[8:9], s[80:81], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 12
+; SI-NEXT: v_writelane_b32 v62, s9, 13
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 22
+; SI-NEXT: v_writelane_b32 v62, s9, 23
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 20
+; SI-NEXT: v_writelane_b32 v62, s9, 21
+; SI-NEXT: s_lshr_b64 s[8:9], s[66:67], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 18
+; SI-NEXT: v_writelane_b32 v62, s9, 19
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 28
+; SI-NEXT: v_writelane_b32 v62, s9, 29
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 26
+; SI-NEXT: v_writelane_b32 v62, s9, 27
+; SI-NEXT: s_lshr_b64 s[8:9], s[52:53], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 24
+; SI-NEXT: v_writelane_b32 v62, s9, 25
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 34
+; SI-NEXT: v_writelane_b32 v62, s9, 35
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 32
+; SI-NEXT: v_writelane_b32 v62, s9, 33
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 30
+; SI-NEXT: v_writelane_b32 v62, s9, 31
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 42
+; SI-NEXT: v_writelane_b32 v62, s9, 43
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 40
+; SI-NEXT: v_writelane_b32 v62, s9, 41
+; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 38
+; SI-NEXT: v_writelane_b32 v62, s9, 39
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 48
+; SI-NEXT: v_writelane_b32 v62, s9, 49
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 46
+; SI-NEXT: v_writelane_b32 v62, s9, 47
+; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 44
+; SI-NEXT: v_writelane_b32 v62, s9, 45
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 54
+; SI-NEXT: v_writelane_b32 v62, s9, 55
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 52
+; SI-NEXT: v_writelane_b32 v62, s9, 53
+; SI-NEXT: s_lshr_b64 s[8:9], s[76:77], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 50
+; SI-NEXT: v_writelane_b32 v62, s9, 51
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 24
+; SI-NEXT: v_writelane_b32 v62, s8, 62
+; SI-NEXT: v_writelane_b32 v62, s9, 63
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 16
+; SI-NEXT: v_writelane_b32 v62, s8, 60
+; SI-NEXT: v_writelane_b32 v62, s9, 61
+; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 8
+; SI-NEXT: v_writelane_b32 v62, s8, 58
+; SI-NEXT: v_writelane_b32 v62, s9, 59
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 4
+; SI-NEXT: v_writelane_b32 v61, s9, 5
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 2
+; SI-NEXT: v_writelane_b32 v61, s9, 3
+; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 0
+; SI-NEXT: v_writelane_b32 v61, s9, 1
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 10
+; SI-NEXT: v_writelane_b32 v61, s9, 11
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 8
+; SI-NEXT: v_writelane_b32 v61, s9, 9
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 6
+; SI-NEXT: v_writelane_b32 v61, s9, 7
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 16
+; SI-NEXT: v_writelane_b32 v61, s9, 17
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 14
+; SI-NEXT: v_writelane_b32 v61, s9, 15
+; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 12
+; SI-NEXT: v_writelane_b32 v61, s9, 13
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 22
+; SI-NEXT: v_writelane_b32 v61, s9, 23
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 20
+; SI-NEXT: v_writelane_b32 v61, s9, 21
+; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 18
+; SI-NEXT: v_writelane_b32 v61, s9, 19
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 28
+; SI-NEXT: v_writelane_b32 v61, s9, 29
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 26
+; SI-NEXT: v_writelane_b32 v61, s9, 27
+; SI-NEXT: s_lshr_b64 s[8:9], s[10:11], 8
+; SI-NEXT: v_writelane_b32 v61, s8, 24
+; SI-NEXT: v_writelane_b32 v61, s9, 25
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 24
+; SI-NEXT: v_writelane_b32 v61, s8, 36
+; SI-NEXT: v_writelane_b32 v61, s9, 37
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
+; SI-NEXT: v_writelane_b32 v61, s8, 34
+; SI-NEXT: v_writelane_b32 v62, s6, 36
+; SI-NEXT: v_writelane_b32 v61, s9, 35
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 8
+; SI-NEXT: v_writelane_b32 v62, s7, 37
+; SI-NEXT: s_mov_b32 s6, s92
+; SI-NEXT: v_writelane_b32 v61, s8, 32
+; SI-NEXT: v_writelane_b32 v62, s6, 56
+; SI-NEXT: v_writelane_b32 v61, s9, 33
+; SI-NEXT: s_lshr_b32 s88, s98, 8
+; SI-NEXT: v_writelane_b32 v62, s7, 57
+; SI-NEXT: s_lshr_b32 s6, s90, 8
; SI-NEXT: .LBB91_5: ; %end
-; SI-NEXT: s_and_b32 s5, s8, 0xff
; SI-NEXT: v_readlane_b32 s8, v62, 0
+; SI-NEXT: s_and_b32 s5, s96, 0xff
; SI-NEXT: v_readlane_b32 s9, v62, 1
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: v_readlane_b32 s8, v62, 2
; SI-NEXT: v_readlane_b32 s9, v62, 3
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 vcc_lo, v62, 4
+; SI-NEXT: v_readlane_b32 s96, v62, 4
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, vcc_lo, 24
+; SI-NEXT: s_lshl_b32 s9, s96, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s96, 0xff
+; SI-NEXT: s_and_b32 s5, s98, 0xff
; SI-NEXT: s_lshl_b32 s8, s88, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: s_and_b32 s8, vcc_hi, 0xff
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
@@ -158440,7 +159360,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
; SI-NEXT: v_readlane_b32 s8, v62, 6
-; SI-NEXT: v_readlane_b32 vcc_hi, v62, 5
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s86, 0xff
; SI-NEXT: v_readlane_b32 s9, v62, 7
@@ -158464,7 +159383,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 8, v0
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s65, 0xff
+; SI-NEXT: s_and_b32 s8, s49, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v24
@@ -158497,7 +159416,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s69, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v19
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v20
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
; SI-NEXT: v_readlane_b32 s8, v62, 18
@@ -158520,12 +159439,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s48, 0xff
-; SI-NEXT: s_lshl_b32 s8, s75, 8
+; SI-NEXT: s_and_b32 s5, s64, 0xff
+; SI-NEXT: s_lshl_b32 s8, s57, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s93, 0xff
+; SI-NEXT: s_and_b32 s8, s75, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v43
@@ -158551,7 +159470,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s98, 0xff
+; SI-NEXT: s_and_b32 s5, s50, 0xff
; SI-NEXT: s_lshl_b32 s8, s58, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v0
@@ -158580,31 +159499,34 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 36, v0
; SI-NEXT: s_or_b32 s5, s5, s8
+; SI-NEXT: v_readlane_b32 s8, v62, 36
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s82, 0xff
-; SI-NEXT: s_lshl_b32 s8, s43, 8
+; SI-NEXT: s_and_b32 s5, s8, 0xff
+; SI-NEXT: s_lshl_b32 s8, s93, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 40, v0
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s91, 0xff
+; SI-NEXT: s_and_b32 s8, s15, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
+; SI-NEXT: v_readlane_b32 s9, v62, 37
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v55
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 36
+; SI-NEXT: v_readlane_b32 s8, v62, 38
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s78, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 37
+; SI-NEXT: v_readlane_b32 s9, v62, 39
; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: v_readlane_b32 s61, v62, 35
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 38
-; SI-NEXT: v_readlane_b32 s9, v62, 39
+; SI-NEXT: v_readlane_b32 s8, v62, 40
+; SI-NEXT: v_readlane_b32 s9, v62, 41
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s42, v62, 40
+; SI-NEXT: v_readlane_b32 s60, v62, 42
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s42, 24
+; SI-NEXT: s_lshl_b32 s9, s60, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -158617,25 +159539,25 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 48, v0
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_and_b32 s8, s57, 0xff
+; SI-NEXT: s_and_b32 s8, s43, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v50
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v45
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 42
+; SI-NEXT: v_readlane_b32 s8, v62, 44
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s94, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 43
+; SI-NEXT: v_readlane_b32 s9, v62, 45
; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s43, v62, 41
+; SI-NEXT: v_readlane_b32 s61, v62, 43
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 44
-; SI-NEXT: v_readlane_b32 s9, v62, 45
+; SI-NEXT: v_readlane_b32 s8, v62, 46
+; SI-NEXT: v_readlane_b32 s9, v62, 47
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s42, v62, 46
+; SI-NEXT: v_readlane_b32 s60, v62, 48
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s42, 24
+; SI-NEXT: s_lshl_b32 s9, s60, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -158643,7 +159565,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s64, 0xff
+; SI-NEXT: s_and_b32 s5, s82, 0xff
; SI-NEXT: s_lshl_b32 s8, s27, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 56, v0
@@ -158651,19 +159573,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s89, 0xff
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v60
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v50
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 48
+; SI-NEXT: v_readlane_b32 s8, v62, 50
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s76, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 49
+; SI-NEXT: v_readlane_b32 s9, v62, 51
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 50
-; SI-NEXT: v_readlane_b32 s9, v62, 51
+; SI-NEXT: v_readlane_b32 s8, v62, 52
+; SI-NEXT: v_readlane_b32 s9, v62, 53
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 52
+; SI-NEXT: v_readlane_b32 s26, v62, 54
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s26, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -158677,21 +159599,23 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v2, s5
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s5, s92, 0xff
+; SI-NEXT: v_readlane_b32 s8, v62, 56
+; SI-NEXT: s_and_b32 s5, s8, 0xff
; SI-NEXT: s_lshl_b32 s8, s28, 8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s73, 0xff
+; SI-NEXT: v_readlane_b32 s9, v62, 57
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_readlane_b32 s27, v62, 53
-; SI-NEXT: v_readlane_b32 s26, v62, 58
+; SI-NEXT: v_readlane_b32 s27, v62, 55
+; SI-NEXT: v_readlane_b32 s26, v62, 62
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v2, vcc, 0x44, v0
-; SI-NEXT: v_readlane_b32 s27, v62, 59
; SI-NEXT: s_and_b32 s4, s4, 0xff
+; SI-NEXT: v_readlane_b32 s97, v62, 5
; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s61, v62, 35
-; SI-NEXT: v_readlane_b32 s43, v62, 47
+; SI-NEXT: v_readlane_b32 s61, v62, 49
+; SI-NEXT: v_readlane_b32 s27, v62, 63
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s99, v63, 33
; SI-NEXT: v_readlane_b32 s98, v63, 32
@@ -158726,14 +159650,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 54
+; SI-NEXT: v_readlane_b32 s8, v62, 58
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s62, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 55
+; SI-NEXT: v_readlane_b32 s9, v62, 59
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 56
-; SI-NEXT: v_readlane_b32 s9, v62, 57
+; SI-NEXT: v_readlane_b32 s8, v62, 60
+; SI-NEXT: v_readlane_b32 s9, v62, 61
; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s26, 24
@@ -158742,8 +159666,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s54, 0xff
-; SI-NEXT: s_lshl_b32 s8, s74, 8
+; SI-NEXT: s_and_b32 s5, s74, 0xff
+; SI-NEXT: s_lshl_b32 s8, s21, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x48, v0
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158753,18 +159677,18 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v34
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 60
+; SI-NEXT: v_readlane_b32 s8, v61, 0
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s46, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 61
+; SI-NEXT: v_readlane_b32 s9, v61, 1
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 62
-; SI-NEXT: v_readlane_b32 s9, v62, 63
+; SI-NEXT: v_readlane_b32 s8, v61, 2
+; SI-NEXT: v_readlane_b32 s9, v61, 3
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v61, 0
+; SI-NEXT: v_readlane_b32 s20, v61, 4
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s20, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -158772,7 +159696,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s68, 0xff
+; SI-NEXT: s_and_b32 s5, s54, 0xff
; SI-NEXT: s_lshl_b32 s8, s22, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x50, v0
@@ -158783,19 +159707,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v31
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 2
+; SI-NEXT: v_readlane_b32 s8, v61, 6
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s40, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 3
+; SI-NEXT: v_readlane_b32 s9, v61, 7
; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v61, 1
+; SI-NEXT: v_readlane_b32 s21, v61, 5
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 4
-; SI-NEXT: v_readlane_b32 s9, v61, 5
+; SI-NEXT: v_readlane_b32 s8, v61, 8
+; SI-NEXT: v_readlane_b32 s9, v61, 9
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v61, 6
+; SI-NEXT: v_readlane_b32 s20, v61, 10
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s20, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_waitcnt expcnt(0)
@@ -158803,8 +159727,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s56, 0xff
-; SI-NEXT: s_lshl_b32 s8, s21, 8
+; SI-NEXT: s_and_b32 s5, s68, 0xff
+; SI-NEXT: s_lshl_b32 s8, s56, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x58, v0
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158814,16 +159738,17 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v30
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 8
+; SI-NEXT: v_readlane_b32 s8, v61, 12
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s24, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 9
+; SI-NEXT: v_readlane_b32 s9, v61, 13
; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: v_readlane_b32 s21, v61, 11
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 10
-; SI-NEXT: v_readlane_b32 s9, v61, 11
+; SI-NEXT: v_readlane_b32 s8, v61, 14
+; SI-NEXT: v_readlane_b32 s9, v61, 15
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 12
+; SI-NEXT: v_readlane_b32 s20, v61, 16
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s20, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -158833,7 +159758,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s36, 0xff
+; SI-NEXT: s_and_b32 s5, s42, 0xff
; SI-NEXT: s_lshl_b32 s8, s18, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x60, v0
@@ -158844,16 +159769,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v29
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 14
+; SI-NEXT: v_readlane_b32 s8, v61, 18
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s16, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 15
+; SI-NEXT: v_readlane_b32 s9, v61, 19
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 16
-; SI-NEXT: v_readlane_b32 s9, v61, 17
+; SI-NEXT: v_readlane_b32 s8, v61, 20
+; SI-NEXT: v_readlane_b32 s9, v61, 21
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s16, v61, 18
+; SI-NEXT: v_readlane_b32 s16, v61, 22
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s16, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -158863,8 +159788,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s38, 0xff
-; SI-NEXT: s_lshl_b32 s8, s15, 8
+; SI-NEXT: s_and_b32 s5, s36, 0xff
+; SI-NEXT: s_lshl_b32 s8, s91, 8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x68, v0
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158874,16 +159799,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v27
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 20
+; SI-NEXT: v_readlane_b32 s8, v61, 24
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s10, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 21
+; SI-NEXT: v_readlane_b32 s9, v61, 25
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 22
-; SI-NEXT: v_readlane_b32 s9, v61, 23
+; SI-NEXT: v_readlane_b32 s8, v61, 26
+; SI-NEXT: v_readlane_b32 s9, v61, 27
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v61, 24
+; SI-NEXT: v_readlane_b32 s10, v61, 28
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s9, s10, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -158893,11 +159818,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: s_and_b32 s5, s14, 0xff
+; SI-NEXT: s_and_b32 s5, s38, 0xff
; SI-NEXT: s_lshl_b32 s8, s12, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 26
-; SI-NEXT: v_readlane_b32 s9, v61, 27
+; SI-NEXT: v_readlane_b32 s8, v61, 30
+; SI-NEXT: v_readlane_b32 s9, v61, 31
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x70, v0
; SI-NEXT: s_and_b32 s8, s9, 0xff
@@ -158905,16 +159830,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v26
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 28
+; SI-NEXT: v_readlane_b32 s8, v61, 32
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_readlane_b32 s9, v61, 29
+; SI-NEXT: v_readlane_b32 s9, v61, 33
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v61, 30
-; SI-NEXT: v_readlane_b32 s9, v61, 31
+; SI-NEXT: v_readlane_b32 s8, v61, 34
+; SI-NEXT: v_readlane_b32 s9, v61, 35
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v61, 32
+; SI-NEXT: v_readlane_b32 s8, v61, 36
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -158928,18 +159853,17 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s4, s90, 0xff
+; SI-NEXT: s_and_b32 s4, s14, 0xff
; SI-NEXT: s_lshl_b32 s5, s6, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s13, 0xff
+; SI-NEXT: s_and_b32 s5, s7, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
-; SI-NEXT: v_readlane_b32 s27, v61, 7
-; SI-NEXT: v_readlane_b32 s21, v61, 13
-; SI-NEXT: v_readlane_b32 s17, v61, 19
-; SI-NEXT: v_readlane_b32 s11, v61, 25
-; SI-NEXT: v_readlane_b32 s9, v61, 33
+; SI-NEXT: v_readlane_b32 s21, v61, 17
+; SI-NEXT: v_readlane_b32 s17, v61, 23
+; SI-NEXT: v_readlane_b32 s11, v61, 29
+; SI-NEXT: v_readlane_b32 s9, v61, 37
; SI-NEXT: v_readlane_b32 s68, v63, 18
; SI-NEXT: v_readlane_b32 s54, v63, 12
; SI-NEXT: v_readlane_b32 s38, v63, 4
@@ -159044,7 +159968,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
; VI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB91_3
+; VI-NEXT: s_cbranch_scc0 .LBB91_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
; VI-NEXT: v_writelane_b32 v62, s46, 2
@@ -159195,13 +160119,175 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: s_lshr_b64 s[78:79], s[6:7], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[8:9], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[44:45], 24
-; VI-NEXT: s_cbranch_execnz .LBB91_4
-; VI-NEXT: .LBB91_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[12:13], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[14:15], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[40:41], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[42:43], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[44:45], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB91_3
+; VI-NEXT: .LBB91_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB91_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s47, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s47, 1
+; VI-NEXT: s_cbranch_scc1 .LBB91_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s46, s45, 16
; VI-NEXT: v_mov_b32_e32 v2, 0x40c00000
; VI-NEXT: v_add_f32_e32 v1, s46, v2
@@ -160020,164 +161106,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v57, 8, v36
; VI-NEXT: v_lshrrev_b32_e32 v36, 16, v53
; VI-NEXT: v_lshrrev_b32_e32 v53, 8, v53
-; VI-NEXT: s_branch .LBB91_5
-; VI-NEXT: .LBB91_3:
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB91_2
-; VI-NEXT: .LBB91_4:
+; VI-NEXT: s_branch .LBB91_6
+; VI-NEXT: .LBB91_5:
; VI-NEXT: v_readlane_b32 s47, v62, 0
; VI-NEXT: v_mov_b32_e32 v1, s47
; VI-NEXT: v_readlane_b32 s47, v62, 1
@@ -160358,25 +161288,25 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s10
-; VI-NEXT: v_mov_b32_e32 v7, s30
+; VI-NEXT: v_mov_b32_e32 v7, vcc_lo
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s11
-; VI-NEXT: v_mov_b32_e32 v7, s34
+; VI-NEXT: v_mov_b32_e32 v7, s30
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s13
-; VI-NEXT: v_mov_b32_e32 v7, s36
+; VI-NEXT: v_mov_b32_e32 v7, s34
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s46
-; VI-NEXT: v_mov_b32_e32 v7, s38
+; VI-NEXT: v_mov_b32_e32 v7, s36
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
@@ -160406,7 +161336,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v3, s60
; VI-NEXT: v_mov_b32_e32 v45, s78
; VI-NEXT: v_mov_b32_e32 v43, s88
-; VI-NEXT: v_mov_b32_e32 v7, s48
+; VI-NEXT: v_mov_b32_e32 v7, s38
; VI-NEXT: v_mov_b32_e32 v55, s47
; VI-NEXT: v_readlane_b32 s47, v62, 57
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
@@ -160453,7 +161383,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v43, s67
; VI-NEXT: v_mov_b32_e32 v7, v20
; VI-NEXT: v_mov_b32_e32 v20, v10
-; VI-NEXT: .LBB91_5: ; %end
+; VI-NEXT: .LBB91_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v6, v33, v6, s4
; VI-NEXT: v_perm_b32 v26, v26, v58, s4
@@ -160906,7 +161836,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB91_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
; GFX9-NEXT: v_writelane_b32 v62, s46, 17
@@ -161051,11 +161981,165 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: s_lshr_b64 s[90:91], s[10:11], 24
; GFX9-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[12:13], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[44:45], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB91_4
-; GFX9-NEXT: .LBB91_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[40:41], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[42:43], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB91_3
+; GFX9-NEXT: .LBB91_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB91_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s46, s45, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v13, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s46, v13
@@ -161839,156 +162923,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; GFX9-NEXT: s_branch .LBB91_5
-; GFX9-NEXT: .LBB91_3:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB91_2
-; GFX9-NEXT: .LBB91_4:
+; GFX9-NEXT: s_branch .LBB91_6
+; GFX9-NEXT: .LBB91_5:
; GFX9-NEXT: v_mov_b32_e32 v1, s28
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v1, s29
@@ -162242,10 +163178,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v48, s72
; GFX9-NEXT: v_mov_b32_e32 v49, s74
; GFX9-NEXT: v_mov_b32_e32 v50, s76
-; GFX9-NEXT: v_mov_b32_e32 v20, s30
-; GFX9-NEXT: v_mov_b32_e32 v31, s34
-; GFX9-NEXT: v_mov_b32_e32 v34, s36
-; GFX9-NEXT: .LBB91_5: ; %end
+; GFX9-NEXT: v_mov_b32_e32 v20, vcc_lo
+; GFX9-NEXT: v_mov_b32_e32 v31, s30
+; GFX9-NEXT: v_mov_b32_e32 v34, s34
+; GFX9-NEXT: .LBB91_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v14, v14, v33, s4
@@ -162709,78 +163645,78 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s104, 0
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB91_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s27, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s68, s5, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s67, s5, 24
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 17
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s27, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s100, s5, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s70, s5, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s104, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s99, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s69, s5, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s103, s4, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 1
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s27, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s80, s4, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s65, s7, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s99, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s71, s4, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s64, s7, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s98, s7, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 18
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s26, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s67, s7, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s103, s6, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s71, s6, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s66, s7, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s102, s6, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s70, s6, 8
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 0
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s26, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s54, s9, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s98, s9, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s64, s9, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s53, s9, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s97, s9, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s55, s9, 8
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s25, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s102, s8, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s69, s8, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s51, s11, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s101, s8, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s68, s8, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s50, s11, 24
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 20
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s25, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s97, s11, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s53, s11, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s101, s10, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s96, s11, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s52, s11, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s100, s10, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 3
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s25, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s66, s10, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s48, s13, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s87, s13, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s65, s10, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s39, s13, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s86, s13, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 21
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s24, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s50, s13, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s96, s12, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s55, s12, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s49, s13, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s87, s12, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s54, s12, 8
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 2
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s24, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s37, s15, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s85, s15, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s39, s15, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s36, s15, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s84, s15, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s38, s15, 8
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 19
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s23, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s86, s14, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s52, s14, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s35, s41, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s85, s14, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s51, s14, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s34, s41, 24
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 23
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s23, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s83, s41, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s36, s41, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s84, s40, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s82, s41, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s35, s41, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s83, s40, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 5
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s23, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s49, s40, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 vcc_hi, s29, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s81, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s48, s40, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s30, s29, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s80, s29, 16
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s22, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s34, s29, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s82, s28, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s38, s28, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s31, s29, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s37, s28, 8
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 4
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s22, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[72:73], s[26:27], 24
@@ -162805,7 +163741,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s20, 16
; GFX11-TRUE16-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[94:95], s[40:41], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[30:31], s[28:29], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 vcc, s[28:29], 24
; GFX11-TRUE16-NEXT: v_writelane_b32 v76, s42, 6
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s20, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -162861,9 +163797,152 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_writelane_b32 v77, s42, 5
; GFX11-TRUE16-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-TRUE16-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB91_3
+; GFX11-TRUE16-NEXT: .LBB91_2:
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: s_mov_b32 s104, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr37
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr81
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr31
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr80
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr30
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr48
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr83
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr35
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr82
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr34
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr51
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr85
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr38
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr84
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr36
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr54
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr87
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr49
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr86
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr39
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr65
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr100
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr52
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr96
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr50
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr68
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr101
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr55
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr97
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr53
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr70
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr102
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr66
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr98
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr64
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr71
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr103
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr69
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr99
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr67
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $vcc_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
+; GFX11-TRUE16-NEXT: .LBB91_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s43, s104, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s42, s29, 16
; GFX11-TRUE16-NEXT: s_and_b32 s29, s29, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s42
@@ -163560,146 +164639,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-TRUE16-NEXT: s_branch .LBB91_5
-; GFX11-TRUE16-NEXT: .LBB91_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr38
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr82
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr34
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr81
-; GFX11-TRUE16-NEXT: ; implicit-def: $vcc_hi
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr49
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr84
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr36
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr83
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr35
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr52
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr86
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr39
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr85
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr37
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr55
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr96
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr50
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr87
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr48
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr66
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr101
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr53
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr97
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr51
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr69
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr102
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr64
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr98
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr54
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr71
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr103
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr67
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr99
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr65
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr80
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr104
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr70
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr100
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr68
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr30
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; kill: killed $sgpr43
-; GFX11-TRUE16-NEXT: s_branch .LBB91_2
-; GFX11-TRUE16-NEXT: .LBB91_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB91_6
+; GFX11-TRUE16-NEXT: .LBB91_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v118, s2 :: v_dual_mov_b32 v115, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v86, s8 :: v_dual_mov_b32 v165, s26
@@ -163727,57 +164668,57 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v163, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v113, s104 :: v_dual_mov_b32 v112, s100
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v103, s103 :: v_dual_mov_b32 v100, s102
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v113, s103 :: v_dual_mov_b32 v112, s99
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v103, s102 :: v_dual_mov_b32 v100, s101
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v161, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v101, s99 :: v_dual_mov_b32 v96, s101
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v97, s98 :: v_dual_mov_b32 v84, s96
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v101, s98 :: v_dual_mov_b32 v96, s100
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v97, s97 :: v_dual_mov_b32 v84, s87
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v160, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v85, s97 :: v_dual_mov_b32 v70, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v83, s12 :: v_dual_mov_b32 v80, s87
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v85, s96 :: v_dual_mov_b32 v70, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v83, s12 :: v_dual_mov_b32 v80, s86
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v149, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v68, s14 :: v_dual_mov_b32 v69, s86
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v68, s14 :: v_dual_mov_b32 v69, s85
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v65, s15 :: v_dual_mov_b32 v64, s40
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v148, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v67, s85 :: v_dual_mov_b32 v66, s84
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s41 :: v_dual_mov_b32 v55, s83
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v67, s84 :: v_dual_mov_b32 v66, s83
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s41 :: v_dual_mov_b32 v55, s82
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v145, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v52, s28 :: v_dual_mov_b32 v53, s82
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v50, s29 :: v_dual_mov_b32 v51, s81
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v52, s28 :: v_dual_mov_b32 v53, s81
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v50, s29 :: v_dual_mov_b32 v51, s80
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v144, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s80 :: v_dual_mov_b32 v19, s70
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s68 :: v_dual_mov_b32 v13, s71
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s71 :: v_dual_mov_b32 v19, s69
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s67 :: v_dual_mov_b32 v13, s70
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v133, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s65 :: v_dual_mov_b32 v14, s67
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s69 :: v_dual_mov_b32 v12, s64
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s64 :: v_dual_mov_b32 v14, s66
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s68 :: v_dual_mov_b32 v12, s55
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v132, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s54 :: v_dual_mov_b32 v28, s51
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s66 :: v_dual_mov_b32 v10, s53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s53 :: v_dual_mov_b32 v28, s50
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s65 :: v_dual_mov_b32 v10, s52
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v129, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s55 :: v_dual_mov_b32 v8, s50
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s48 :: v_dual_mov_b32 v6, s39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s54 :: v_dual_mov_b32 v8, s49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s39 :: v_dual_mov_b32 v6, s38
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v128, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s52 :: v_dual_mov_b32 v48, s35
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s37 :: v_dual_mov_b32 v36, s36
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s51 :: v_dual_mov_b32 v48, s34
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s36 :: v_dual_mov_b32 v36, s35
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v117, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, s49 :: v_dual_mov_b32 v44, s38
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, vcc_hi :: v_dual_mov_b32 v4, s34
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, s48 :: v_dual_mov_b32 v44, s37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, s30 :: v_dual_mov_b32 v4, s31
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v178, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 17
@@ -163796,7 +164737,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 20
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s90 :: v_dual_mov_b32 v23, s92
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s94
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s30
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v181, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v76, 21
@@ -163846,7 +164787,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v63, s0
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v72, s0
-; GFX11-TRUE16-NEXT: .LBB91_5: ; %end
+; GFX11-TRUE16-NEXT: .LBB91_6: ; %end
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v128, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v38, v115, v73, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v115, v117, v63, 0xc0c0004
@@ -164159,78 +165100,78 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s104, 0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB91_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s27, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s68, s5, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s67, s5, 24
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 17
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s27, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s100, s5, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s70, s5, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s104, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s99, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s69, s5, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s103, s4, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 1
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s27, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s80, s4, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s65, s7, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s99, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s71, s4, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s64, s7, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s98, s7, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 18
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s26, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s67, s7, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s103, s6, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s71, s6, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s66, s7, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s102, s6, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s70, s6, 8
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 0
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s26, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s54, s9, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s98, s9, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s64, s9, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s53, s9, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s97, s9, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s55, s9, 8
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s25, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s102, s8, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s69, s8, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s51, s11, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s101, s8, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s68, s8, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s50, s11, 24
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 20
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s25, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s97, s11, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s53, s11, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s101, s10, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s96, s11, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s52, s11, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s100, s10, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 3
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s25, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s66, s10, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s48, s13, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s87, s13, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s65, s10, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s39, s13, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s86, s13, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 21
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s24, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s50, s13, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s96, s12, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s55, s12, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s49, s13, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s87, s12, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s54, s12, 8
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 2
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s24, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s37, s15, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s85, s15, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s39, s15, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s36, s15, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s84, s15, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s38, s15, 8
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 19
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s23, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s86, s14, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s52, s14, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s35, s41, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s85, s14, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s51, s14, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s34, s41, 24
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 23
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s23, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s83, s41, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s36, s41, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s84, s40, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s82, s41, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s35, s41, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s83, s40, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 5
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s23, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s49, s40, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 vcc_hi, s29, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s81, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s48, s40, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s30, s29, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s80, s29, 16
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s22, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s34, s29, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s82, s28, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s38, s28, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s31, s29, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s81, s28, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s37, s28, 8
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 4
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s22, 8
; GFX11-FAKE16-NEXT: s_lshr_b64 s[72:73], s[26:27], 24
@@ -164255,7 +165196,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s20, 16
; GFX11-FAKE16-NEXT: s_lshr_b64 s[92:93], s[14:15], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[94:95], s[40:41], 24
-; GFX11-FAKE16-NEXT: s_lshr_b64 s[30:31], s[28:29], 24
+; GFX11-FAKE16-NEXT: s_lshr_b64 vcc, s[28:29], 24
; GFX11-FAKE16-NEXT: v_writelane_b32 v76, s42, 6
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s20, 8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -164311,9 +165252,152 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_writelane_b32 v77, s42, 5
; GFX11-FAKE16-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-FAKE16-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB91_3
+; GFX11-FAKE16-NEXT: .LBB91_2:
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: s_mov_b32 s104, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr37
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr81
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr31
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr80
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr30
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr48
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr83
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr35
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr82
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr34
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr51
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr85
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr38
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr84
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr36
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr54
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr87
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr49
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr86
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr39
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr65
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr100
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr52
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr96
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr50
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr68
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr101
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr55
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr97
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr53
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr70
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr102
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr66
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr98
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr64
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr71
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr103
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr69
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr99
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr67
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $vcc_lo
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
+; GFX11-FAKE16-NEXT: .LBB91_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s43, s104, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s42, s29, 16
; GFX11-FAKE16-NEXT: s_and_b32 s29, s29, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s42
@@ -165030,146 +166114,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 24, v6
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-FAKE16-NEXT: s_branch .LBB91_5
-; GFX11-FAKE16-NEXT: .LBB91_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr38
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr82
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr34
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr81
-; GFX11-FAKE16-NEXT: ; implicit-def: $vcc_hi
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr49
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr84
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr36
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr83
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr35
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr52
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr86
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr39
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr85
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr37
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr55
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr96
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr50
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr87
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr48
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr66
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr101
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr53
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr97
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr51
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr69
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr102
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr64
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr98
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr54
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr71
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr103
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr67
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr99
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr65
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr80
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr104
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr70
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr100
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr68
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr30
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; kill: killed $sgpr43
-; GFX11-FAKE16-NEXT: s_branch .LBB91_2
-; GFX11-FAKE16-NEXT: .LBB91_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB91_6
+; GFX11-FAKE16-NEXT: .LBB91_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v112, s0 :: v_dual_mov_b32 v103, s1
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v160, s26 :: v_dual_mov_b32 v151, s27
@@ -165196,56 +166142,56 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v162, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v71, s10 :: v_dual_mov_b32 v68, s11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v102, s104 :: v_dual_mov_b32 v101, s100
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v102, s103 :: v_dual_mov_b32 v101, s99
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v149, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 6
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v100, s103 :: v_dual_mov_b32 v97, s102
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v98, s99 :: v_dual_mov_b32 v85, s101
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v100, s102 :: v_dual_mov_b32 v97, s101
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v98, s98 :: v_dual_mov_b32 v85, s100
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v150, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v86, s98 :: v_dual_mov_b32 v81, s96
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, s97 :: v_dual_mov_b32 v67, s13
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v86, s97 :: v_dual_mov_b32 v81, s87
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, s96 :: v_dual_mov_b32 v67, s13
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v145, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 8
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, s12 :: v_dual_mov_b32 v69, s87
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v65, s14 :: v_dual_mov_b32 v66, s86
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, s12 :: v_dual_mov_b32 v69, s86
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v65, s14 :: v_dual_mov_b32 v66, s85
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v147, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v54, s15 :: v_dual_mov_b32 v53, s40
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v64, s85 :: v_dual_mov_b32 v55, s84
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v64, s84 :: v_dual_mov_b32 v55, s83
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v133, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 10
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s41 :: v_dual_mov_b32 v52, s83
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, s28 :: v_dual_mov_b32 v50, s82
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s41 :: v_dual_mov_b32 v52, s82
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, s28 :: v_dual_mov_b32 v50, s81
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v134, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, s29 :: v_dual_mov_b32 v48, s81
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s80 :: v_dual_mov_b32 v23, s65
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, s29 :: v_dual_mov_b32 v48, s80
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s71 :: v_dual_mov_b32 v23, s64
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v130, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 12
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s71 :: v_dual_mov_b32 v17, s67
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v11, s69 :: v_dual_mov_b32 v24, s54
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s70 :: v_dual_mov_b32 v17, s66
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v11, s68 :: v_dual_mov_b32 v24, s53
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v129, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 13
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s64 :: v_dual_mov_b32 v9, s66
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s51 :: v_dual_mov_b32 v7, s55
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v12, s55 :: v_dual_mov_b32 v9, s65
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s50 :: v_dual_mov_b32 v7, s54
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v118, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 14
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s53 :: v_dual_mov_b32 v31, s48
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s50 :: v_dual_mov_b32 v5, s52
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s52 :: v_dual_mov_b32 v31, s39
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s49 :: v_dual_mov_b32 v5, s51
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v117, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 15
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s37 :: v_dual_mov_b32 v6, s39
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v43, s49 :: v_dual_mov_b32 v38, s36
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s36 :: v_dual_mov_b32 v6, s38
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v43, s48 :: v_dual_mov_b32 v38, s35
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v114, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 16
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v41, s35 :: v_dual_mov_b32 v44, s38
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v45, vcc_hi :: v_dual_mov_b32 v4, s34
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v41, s34 :: v_dual_mov_b32 v44, s37
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v45, s30 :: v_dual_mov_b32 v4, s31
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v176, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 17
@@ -165264,12 +166210,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v20, s90 :: v_dual_mov_b32 v21, s92
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s94
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s30
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v179, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 21
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s68
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s70
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v29, s67
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v15, s69
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v177, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v76, 22
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -165316,7 +166262,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v63, s0
; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v72, s0
-; GFX11-FAKE16-NEXT: .LBB91_5: ; %end
+; GFX11-FAKE16-NEXT: .LBB91_6: ; %end
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v117, v35, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v35, v112, v73, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v112, v114, v63, 0xc0c0004
@@ -171934,361 +172880,363 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s4, v24
; SI-NEXT: v_writelane_b32 v47, s4, 20
; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:332
; SI-NEXT: v_writelane_b32 v47, s4, 21
; SI-NEXT: v_readfirstlane_b32 s4, v22
; SI-NEXT: v_writelane_b32 v47, s4, 22
; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:328
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:332
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:324
+; SI-NEXT: v_readfirstlane_b32 s92, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:328
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:320
; SI-NEXT: v_writelane_b32 v47, s4, 23
; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:320
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:316
; SI-NEXT: v_writelane_b32 v47, s4, 24
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:316
-; SI-NEXT: v_writelane_b32 v47, s4, 25
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v47, s4, 26
; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_writelane_b32 v47, s4, 27
+; SI-NEXT: v_writelane_b32 v47, s4, 25
; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v47, s4, 28
+; SI-NEXT: v_writelane_b32 v47, s4, 26
; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: v_writelane_b32 v47, s4, 29
+; SI-NEXT: v_writelane_b32 v47, s4, 27
; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: v_writelane_b32 v47, s4, 30
+; SI-NEXT: v_writelane_b32 v47, s4, 28
; SI-NEXT: v_readfirstlane_b32 s4, v13
-; SI-NEXT: v_writelane_b32 v47, s4, 31
-; SI-NEXT: v_readfirstlane_b32 s4, v12
-; SI-NEXT: v_writelane_b32 v47, s4, 32
-; SI-NEXT: v_readfirstlane_b32 s4, v11
-; SI-NEXT: v_writelane_b32 v47, s4, 33
-; SI-NEXT: v_readfirstlane_b32 s4, v10
-; SI-NEXT: v_writelane_b32 v47, s4, 34
+; SI-NEXT: v_writelane_b32 v47, s4, 29
; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: v_writelane_b32 v47, s4, 35
+; SI-NEXT: v_writelane_b32 v47, s4, 30
; SI-NEXT: v_readfirstlane_b32 s4, v8
-; SI-NEXT: v_writelane_b32 v47, s4, 36
+; SI-NEXT: v_writelane_b32 v47, s4, 31
; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: v_writelane_b32 v47, s4, 37
+; SI-NEXT: v_writelane_b32 v47, s4, 32
; SI-NEXT: v_readfirstlane_b32 s4, v6
-; SI-NEXT: v_writelane_b32 v47, s4, 38
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:308
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:304
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:300
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292
+; SI-NEXT: v_readfirstlane_b32 s7, v18
+; SI-NEXT: v_writelane_b32 v47, s4, 33
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
+; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:280
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264
+; SI-NEXT: v_readfirstlane_b32 s11, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:276
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264
-; SI-NEXT: v_writelane_b32 v47, s4, 39
-; SI-NEXT: v_readfirstlane_b32 s4, v4
-; SI-NEXT: v_writelane_b32 v47, s4, 40
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: v_writelane_b32 v47, s4, 41
+; SI-NEXT: v_readfirstlane_b32 s12, v4
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
+; SI-NEXT: v_readfirstlane_b32 s9, v3
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268
-; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s14, v2
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:308
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:304
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292
+; SI-NEXT: v_readfirstlane_b32 s13, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:300
+; SI-NEXT: v_readfirstlane_b32 s8, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296
; SI-NEXT: ; implicit-def: $vgpr46 : SGPR spill to VGPR lane
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v23
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v21
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256
+; SI-NEXT: v_writelane_b32 v47, s4, 34
+; SI-NEXT: v_readfirstlane_b32 s4, v19
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:232
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248
-; SI-NEXT: v_writelane_b32 v47, s4, 42
-; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
-; SI-NEXT: v_readfirstlane_b32 s83, v21
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:236
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:232
-; SI-NEXT: v_readfirstlane_b32 s86, v22
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:224
-; SI-NEXT: v_readfirstlane_b32 s99, v20
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:248
+; SI-NEXT: v_writelane_b32 v47, s4, 35
+; SI-NEXT: v_readfirstlane_b32 s4, v22
+; SI-NEXT: v_writelane_b32 v47, s4, 36
+; SI-NEXT: v_readfirstlane_b32 s4, v24
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:216
-; SI-NEXT: v_readfirstlane_b32 s17, v19
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216
+; SI-NEXT: v_readfirstlane_b32 s19, v20
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:208
-; SI-NEXT: v_writelane_b32 v47, s4, 43
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:208
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:228
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:224
+; SI-NEXT: v_writelane_b32 v47, s4, 37
+; SI-NEXT: v_readfirstlane_b32 s84, v18
+; SI-NEXT: v_readfirstlane_b32 s95, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s66, v14
-; SI-NEXT: v_readfirstlane_b32 s84, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:188
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
-; SI-NEXT: v_readfirstlane_b32 s9, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
-; SI-NEXT: v_readfirstlane_b32 s16, v10
-; SI-NEXT: v_readfirstlane_b32 s46, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
-; SI-NEXT: v_readfirstlane_b32 s24, v8
+; SI-NEXT: v_readfirstlane_b32 s68, v8
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
-; SI-NEXT: v_readfirstlane_b32 s29, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
-; SI-NEXT: v_readfirstlane_b32 s10, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
-; SI-NEXT: v_readfirstlane_b32 s13, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
-; SI-NEXT: v_readfirstlane_b32 s19, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:164
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
-; SI-NEXT: v_readfirstlane_b32 s47, v5
+; SI-NEXT: v_readfirstlane_b32 s44, v6
+; SI-NEXT: v_readfirstlane_b32 s26, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT: v_readfirstlane_b32 s45, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:128
-; SI-NEXT: v_readfirstlane_b32 s27, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:108
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s74, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
+; SI-NEXT: v_readfirstlane_b32 s93, v3
; SI-NEXT: v_readfirstlane_b32 s28, v2
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
-; SI-NEXT: v_readfirstlane_b32 s54, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
-; SI-NEXT: v_readfirstlane_b32 s59, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
-; SI-NEXT: v_readfirstlane_b32 s61, v18
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
+; SI-NEXT: v_readfirstlane_b32 s29, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
+; SI-NEXT: v_readfirstlane_b32 s64, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
+; SI-NEXT: v_readfirstlane_b32 s22, v16
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: v_writelane_b32 v47, s4, 38
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
+; SI-NEXT: v_readfirstlane_b32 s24, v9
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:160
+; SI-NEXT: v_readfirstlane_b32 s75, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
+; SI-NEXT: v_readfirstlane_b32 s99, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64
-; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: v_readfirstlane_b32 s52, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:44
+; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_readfirstlane_b32 s66, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
+; SI-NEXT: v_readfirstlane_b32 s81, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
-; SI-NEXT: v_readfirstlane_b32 s53, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:36
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
-; SI-NEXT: v_readfirstlane_b32 s45, v22
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:28
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
-; SI-NEXT: v_readfirstlane_b32 s60, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
+; SI-NEXT: v_readfirstlane_b32 s57, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; SI-NEXT: v_readfirstlane_b32 s83, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8
-; SI-NEXT: v_readfirstlane_b32 s62, v29
+; SI-NEXT: v_readfirstlane_b32 s86, v29
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
-; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
-; SI-NEXT: v_readfirstlane_b32 s15, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
-; SI-NEXT: v_readfirstlane_b32 s56, v25
+; SI-NEXT: v_readfirstlane_b32 s80, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
+; SI-NEXT: v_readfirstlane_b32 s65, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:28
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
+; SI-NEXT: v_readfirstlane_b32 s46, v25
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16
-; SI-NEXT: v_writelane_b32 v46, s54, 0
-; SI-NEXT: v_writelane_b32 v46, s53, 1
-; SI-NEXT: v_readfirstlane_b32 s40, v19
-; SI-NEXT: v_readfirstlane_b32 s50, v13
-; SI-NEXT: v_readfirstlane_b32 s64, v27
-; SI-NEXT: v_readfirstlane_b32 s85, v12
-; SI-NEXT: v_readfirstlane_b32 s58, v28
-; SI-NEXT: v_readfirstlane_b32 s57, v11
-; SI-NEXT: v_readfirstlane_b32 s11, v30
-; SI-NEXT: v_readfirstlane_b32 s7, v6
-; SI-NEXT: v_writelane_b32 v46, s7, 2
-; SI-NEXT: v_writelane_b32 v46, s64, 3
-; SI-NEXT: v_writelane_b32 v46, s44, 4
-; SI-NEXT: v_writelane_b32 v46, s85, 5
-; SI-NEXT: v_writelane_b32 v46, s8, 6
-; SI-NEXT: v_writelane_b32 v46, s45, 7
-; SI-NEXT: v_writelane_b32 v46, s46, 8
-; SI-NEXT: v_writelane_b32 v46, s47, 9
-; SI-NEXT: v_writelane_b32 v46, s56, 10
-; SI-NEXT: v_writelane_b32 v46, s9, 11
-; SI-NEXT: v_writelane_b32 v46, s57, 12
-; SI-NEXT: v_writelane_b32 v46, s58, 13
-; SI-NEXT: v_writelane_b32 v46, s10, 14
+; SI-NEXT: v_writelane_b32 v47, s4, 39
+; SI-NEXT: v_readfirstlane_b32 s67, v20
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v0
+; SI-NEXT: v_writelane_b32 v46, vcc_hi, 0
+; SI-NEXT: v_readfirstlane_b32 s34, v53
+; SI-NEXT: v_readfirstlane_b32 s55, v8
+; SI-NEXT: v_readfirstlane_b32 s90, v33
+; SI-NEXT: v_readfirstlane_b32 s27, v11
+; SI-NEXT: v_readfirstlane_b32 s36, v39
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s92, v3
-; SI-NEXT: v_writelane_b32 v46, s11, 15
-; SI-NEXT: v_readfirstlane_b32 s12, v10
-; SI-NEXT: v_readfirstlane_b32 s30, v37
-; SI-NEXT: v_writelane_b32 v46, s59, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v7
-; SI-NEXT: v_writelane_b32 v46, s12, 17
-; SI-NEXT: v_readfirstlane_b32 s48, v33
-; SI-NEXT: v_writelane_b32 v46, s60, 18
-; SI-NEXT: v_writelane_b32 v46, s61, 19
-; SI-NEXT: v_writelane_b32 v46, s13, 20
-; SI-NEXT: v_writelane_b32 v46, s62, 21
-; SI-NEXT: v_readfirstlane_b32 s6, v34
-; SI-NEXT: v_writelane_b32 v46, s14, 22
-; SI-NEXT: v_writelane_b32 v46, s15, 23
-; SI-NEXT: v_readfirstlane_b32 s41, v14
-; SI-NEXT: v_readfirstlane_b32 s51, v26
-; SI-NEXT: v_readfirstlane_b32 s87, v31
-; SI-NEXT: v_readfirstlane_b32 s23, v9
-; SI-NEXT: v_readfirstlane_b32 s25, v32
-; SI-NEXT: v_readfirstlane_b32 s26, v8
-; SI-NEXT: v_readfirstlane_b32 s4, v51
-; SI-NEXT: v_writelane_b32 v47, s4, 44
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v47, s4, 45
-; SI-NEXT: v_readfirstlane_b32 s4, v52
-; SI-NEXT: v_writelane_b32 v47, s4, 46
-; SI-NEXT: v_readfirstlane_b32 s4, v55
-; SI-NEXT: v_writelane_b32 v47, s4, 47
-; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: v_writelane_b32 v47, s4, 48
-; SI-NEXT: v_readfirstlane_b32 s4, v40
-; SI-NEXT: v_writelane_b32 v47, s4, 49
+; SI-NEXT: v_readfirstlane_b32 s50, v35
+; SI-NEXT: v_readfirstlane_b32 s49, v5
+; SI-NEXT: v_readfirstlane_b32 s51, v36
+; SI-NEXT: v_readfirstlane_b32 s52, v4
+; SI-NEXT: v_readfirstlane_b32 s54, v37
+; SI-NEXT: v_readfirstlane_b32 s73, v6
+; SI-NEXT: v_readfirstlane_b32 s35, v2
+; SI-NEXT: v_readfirstlane_b32 s37, v48
+; SI-NEXT: v_readfirstlane_b32 s53, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v7
+; SI-NEXT: v_readfirstlane_b32 s72, v34
+; SI-NEXT: v_readfirstlane_b32 s21, v15
+; SI-NEXT: v_readfirstlane_b32 s87, v26
+; SI-NEXT: v_readfirstlane_b32 s70, v38
+; SI-NEXT: v_readfirstlane_b32 s71, v16
+; SI-NEXT: v_readfirstlane_b32 s82, v27
+; SI-NEXT: v_readfirstlane_b32 s4, v32
+; SI-NEXT: v_writelane_b32 v47, s4, 40
+; SI-NEXT: v_writelane_b32 v47, s13, 41
+; SI-NEXT: v_readfirstlane_b32 s79, v13
+; SI-NEXT: v_writelane_b32 v47, s11, 42
+; SI-NEXT: v_writelane_b32 v47, s12, 43
+; SI-NEXT: v_writelane_b32 v47, s6, 44
+; SI-NEXT: v_writelane_b32 v47, s7, 45
+; SI-NEXT: v_writelane_b32 v47, s8, 46
+; SI-NEXT: v_writelane_b32 v47, s9, 47
+; SI-NEXT: v_readfirstlane_b32 s47, v12
+; SI-NEXT: v_writelane_b32 v47, s10, 48
+; SI-NEXT: v_readfirstlane_b32 s56, v28
+; SI-NEXT: v_writelane_b32 v47, s14, 49
+; SI-NEXT: v_readfirstlane_b32 s89, v30
+; SI-NEXT: v_readfirstlane_b32 s91, v10
+; SI-NEXT: v_readfirstlane_b32 s77, v18
+; SI-NEXT: v_readfirstlane_b32 s25, v31
+; SI-NEXT: v_readfirstlane_b32 s98, v9
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v52
+; SI-NEXT: v_readfirstlane_b32 s30, v21
+; SI-NEXT: v_writelane_b32 v46, s30, 1
+; SI-NEXT: v_writelane_b32 v46, s34, 2
+; SI-NEXT: v_writelane_b32 v46, s35, 3
+; SI-NEXT: v_readfirstlane_b32 s60, v54
+; SI-NEXT: v_writelane_b32 v46, s36, 4
+; SI-NEXT: v_readfirstlane_b32 s61, v19
+; SI-NEXT: v_writelane_b32 v46, s60, 5
+; SI-NEXT: v_writelane_b32 v46, s61, 6
+; SI-NEXT: v_readfirstlane_b32 s38, v17
+; SI-NEXT: v_writelane_b32 v46, s37, 7
+; SI-NEXT: s_waitcnt vmcnt(13)
+; SI-NEXT: v_readfirstlane_b32 s39, v1
+; SI-NEXT: v_writelane_b32 v46, s38, 8
+; SI-NEXT: v_readfirstlane_b32 s48, v49
+; SI-NEXT: v_writelane_b32 v46, s39, 9
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s4, v41
-; SI-NEXT: v_writelane_b32 v47, s4, 50
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s4, v43
-; SI-NEXT: v_writelane_b32 v47, s4, 51
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_writelane_b32 v47, s4, 52
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s4, v44
-; SI-NEXT: v_writelane_b32 v47, s4, 53
-; SI-NEXT: v_writelane_b32 v47, s92, 54
-; SI-NEXT: v_writelane_b32 v47, s30, 55
-; SI-NEXT: v_writelane_b32 v47, s36, 56
-; SI-NEXT: v_writelane_b32 v47, s48, 57
-; SI-NEXT: v_writelane_b32 v47, s66, 58
-; SI-NEXT: v_writelane_b32 v47, s52, 59
-; SI-NEXT: v_writelane_b32 v47, s84, 60
-; SI-NEXT: v_writelane_b32 v47, s6, 61
-; SI-NEXT: v_readfirstlane_b32 s67, v35
-; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s65, v36
-; SI-NEXT: v_readfirstlane_b32 s22, v4
-; SI-NEXT: v_readfirstlane_b32 s18, v38
-; SI-NEXT: v_readfirstlane_b32 s20, v15
-; SI-NEXT: v_readfirstlane_b32 s35, v39
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s78, v48
-; SI-NEXT: v_readfirstlane_b32 s98, v16
-; SI-NEXT: v_readfirstlane_b32 s97, v49
+; SI-NEXT: v_readfirstlane_b32 s62, v50
+; SI-NEXT: v_writelane_b32 v46, s48, 10
+; SI-NEXT: v_writelane_b32 v46, s62, 11
+; SI-NEXT: v_writelane_b32 v46, s49, 12
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_readfirstlane_b32 s63, v14
+; SI-NEXT: v_writelane_b32 v46, s50, 13
+; SI-NEXT: v_writelane_b32 v46, s63, 14
+; SI-NEXT: v_writelane_b32 v46, s51, 15
+; SI-NEXT: v_writelane_b32 v46, s52, 16
+; SI-NEXT: v_writelane_b32 v46, s53, 17
+; SI-NEXT: v_writelane_b32 v46, s54, 18
+; SI-NEXT: v_writelane_b32 v46, s55, 19
+; SI-NEXT: v_writelane_b32 v46, s64, 20
+; SI-NEXT: v_writelane_b32 v46, s81, 21
+; SI-NEXT: v_writelane_b32 v46, s72, 22
+; SI-NEXT: v_writelane_b32 v46, s71, 23
+; SI-NEXT: v_writelane_b32 v46, s87, 24
+; SI-NEXT: v_writelane_b32 v46, s80, 25
+; SI-NEXT: v_writelane_b32 v46, s73, 26
+; SI-NEXT: v_writelane_b32 v46, s82, 27
+; SI-NEXT: v_writelane_b32 v46, s79, 28
+; SI-NEXT: v_writelane_b32 v46, s75, 29
+; SI-NEXT: v_writelane_b32 v46, s65, 30
+; SI-NEXT: v_writelane_b32 v46, s44, 31
+; SI-NEXT: v_writelane_b32 v46, s45, 32
+; SI-NEXT: v_writelane_b32 v46, s46, 33
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s55, v1
+; SI-NEXT: v_readfirstlane_b32 s16, v29
+; SI-NEXT: v_writelane_b32 v46, s47, 34
+; SI-NEXT: v_readfirstlane_b32 s17, v43
+; SI-NEXT: v_writelane_b32 v47, s16, 50
+; SI-NEXT: v_writelane_b32 v46, s56, 35
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s81, v50
-; SI-NEXT: v_readfirstlane_b32 s71, v17
+; SI-NEXT: v_readfirstlane_b32 s15, v44
+; SI-NEXT: v_writelane_b32 v47, s17, 51
+; SI-NEXT: v_writelane_b32 v46, s74, 36
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s96, v0
+; SI-NEXT: v_readfirstlane_b32 s18, v23
+; SI-NEXT: v_writelane_b32 v47, s15, 52
+; SI-NEXT: v_writelane_b32 v46, s89, 37
+; SI-NEXT: v_readfirstlane_b32 s23, v55
+; SI-NEXT: v_writelane_b32 v47, s18, 53
+; SI-NEXT: v_writelane_b32 v46, s57, 38
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s79, v53
+; SI-NEXT: v_readfirstlane_b32 s40, v40
+; SI-NEXT: v_writelane_b32 v47, s23, 54
+; SI-NEXT: v_writelane_b32 v46, s91, 39
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s90, v23
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s37, v54
-; SI-NEXT: v_readfirstlane_b32 s82, v21
-; SI-NEXT: v_readfirstlane_b32 s80, v22
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s70, v25
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s76, v42
-; SI-NEXT: v_readfirstlane_b32 s21, v20
-; SI-NEXT: v_writelane_b32 v47, s50, 62
-; SI-NEXT: v_writelane_b32 v46, s40, 24
-; SI-NEXT: v_writelane_b32 v47, s51, 63
-; SI-NEXT: v_writelane_b32 v46, s41, 25
-; SI-NEXT: s_cbranch_scc0 .LBB93_4
+; SI-NEXT: v_readfirstlane_b32 s41, v22
+; SI-NEXT: v_writelane_b32 v47, s40, 55
+; SI-NEXT: v_writelane_b32 v46, s83, 40
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s42, v25
+; SI-NEXT: v_writelane_b32 v47, s41, 56
+; SI-NEXT: v_writelane_b32 v46, s84, 41
+; SI-NEXT: v_readfirstlane_b32 s43, v41
+; SI-NEXT: v_writelane_b32 v47, s42, 57
+; SI-NEXT: v_writelane_b32 v46, s93, 42
+; SI-NEXT: v_writelane_b32 v47, s43, 58
+; SI-NEXT: v_writelane_b32 v46, s86, 43
+; SI-NEXT: v_readfirstlane_b32 s78, v51
+; SI-NEXT: v_writelane_b32 v47, s77, 59
+; SI-NEXT: v_writelane_b32 v46, s95, 44
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s58, v42
+; SI-NEXT: v_writelane_b32 v47, s78, 60
+; SI-NEXT: v_writelane_b32 v46, s66, 45
+; SI-NEXT: v_readfirstlane_b32 s59, v24
+; SI-NEXT: v_writelane_b32 v47, s58, 61
+; SI-NEXT: v_writelane_b32 v46, s67, 46
+; SI-NEXT: v_writelane_b32 v47, s59, 62
+; SI-NEXT: v_writelane_b32 v46, s21, 47
+; SI-NEXT: v_writelane_b32 v47, vcc_lo, 63
+; SI-NEXT: v_writelane_b32 v46, s31, 48
+; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_readlane_b32 s4, v47, 13
; SI-NEXT: v_readlane_b32 s5, v47, 12
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 26
+; SI-NEXT: s_or_b32 s20, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 5
; SI-NEXT: v_readlane_b32 s5, v47, 4
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readlane_b32 s5, v47, 41
-; SI-NEXT: v_writelane_b32 v46, s4, 28
-; SI-NEXT: s_and_b32 s4, s69, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: v_writelane_b32 v46, s69, 44
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 30
-; SI-NEXT: v_readlane_b32 s4, v47, 34
-; SI-NEXT: v_readlane_b32 s5, v47, 33
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: v_readlane_b32 s4, v47, 26
-; SI-NEXT: v_readlane_b32 s5, v47, 25
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 34
+; SI-NEXT: s_or_b32 s88, s4, s5
+; SI-NEXT: s_and_b32 s4, s14, 0xff
+; SI-NEXT: s_lshl_b32 s5, s9, 8
+; SI-NEXT: s_or_b32 s94, s4, s5
+; SI-NEXT: s_and_b32 s4, s8, 0xff
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: s_or_b32 s85, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xff
+; SI-NEXT: s_lshl_b32 s5, s92, 8
+; SI-NEXT: s_or_b32 s96, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 18
; SI-NEXT: v_readlane_b32 s5, v47, 17
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s5, s76, 8
-; SI-NEXT: v_writelane_b32 v46, s24, 45
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: s_and_b32 s4, s82, 0xff
-; SI-NEXT: s_lshl_b32 s5, s37, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: s_and_b32 s4, s71, 0xff
-; SI-NEXT: s_lshl_b32 s5, s81, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 42
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s18, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 46
-; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: s_lshl_b32 s5, s6, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 47
-; SI-NEXT: s_and_b32 s4, s12, 0xff
-; SI-NEXT: s_lshl_b32 s5, s11, 8
+; SI-NEXT: v_writelane_b32 v46, s22, 49
+; SI-NEXT: s_or_b32 s22, s4, s5
+; SI-NEXT: s_and_b32 s4, s59, 0xff
+; SI-NEXT: s_lshl_b32 s5, s58, 8
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s61, 0xff
+; SI-NEXT: s_lshl_b32 s5, s60, 8
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s63, 0xff
+; SI-NEXT: s_lshl_b32 s5, s62, 8
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xff
+; SI-NEXT: s_lshl_b32 s5, s70, 8
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xff
+; SI-NEXT: s_lshl_b32 s5, s72, 8
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xff
+; SI-NEXT: s_lshl_b32 s5, s89, 8
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_and_b32 s4, s41, 0xff
-; SI-NEXT: s_lshl_b32 s5, s40, 8
+; SI-NEXT: s_and_b32 s4, s21, 0xff
+; SI-NEXT: s_lshl_b32 s5, s67, 8
; SI-NEXT: s_or_b32 s72, s4, s5
-; SI-NEXT: s_and_b32 s4, s15, 0xff
-; SI-NEXT: s_lshl_b32 s5, s14, 8
+; SI-NEXT: s_and_b32 s4, s66, 0xff
+; SI-NEXT: s_lshl_b32 s5, s95, 8
; SI-NEXT: s_or_b32 s73, s4, s5
-; SI-NEXT: s_and_b32 s4, s13, 0xff
-; SI-NEXT: s_lshl_b32 s5, s10, 8
+; SI-NEXT: s_and_b32 s4, s93, 0xff
+; SI-NEXT: s_lshl_b32 s5, s74, 8
; SI-NEXT: s_or_b32 s74, s4, s5
-; SI-NEXT: s_and_b32 s4, s9, 0xff
-; SI-NEXT: s_lshl_b32 s5, s8, 8
+; SI-NEXT: s_and_b32 s4, s99, 0xff
+; SI-NEXT: s_lshl_b32 s5, s75, 8
; SI-NEXT: s_or_b32 s75, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 9
; SI-NEXT: v_readlane_b32 s5, v47, 8
@@ -172300,61 +173248,63 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: v_readlane_b32 s6, v47, 6
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s6, s6, 24
-; SI-NEXT: s_or_b32 s68, s6, s4
+; SI-NEXT: s_or_b32 s7, s6, s4
; SI-NEXT: v_readlane_b32 s4, v47, 11
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: v_readlane_b32 s6, v47, 10
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_writelane_b32 v46, s7, 50
; SI-NEXT: s_or_b32 s4, s6, s4
+; SI-NEXT: s_or_b32 s5, s5, s7
; SI-NEXT: v_readlane_b32 s6, v47, 1
; SI-NEXT: v_readlane_b32 s7, v47, 0
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
; SI-NEXT: s_or_b32 s7, s6, s7
-; SI-NEXT: v_readlane_b32 s6, v47, 43
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v47, 42
+; SI-NEXT: s_and_b32 s6, s10, 0xff
+; SI-NEXT: v_readlane_b32 s8, v47, 34
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_or_b32 s91, s8, s6
+; SI-NEXT: s_or_b32 s9, s8, s6
; SI-NEXT: v_readlane_b32 s6, v47, 3
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: v_readlane_b32 s8, v47, 2
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: v_writelane_b32 v46, s9, 51
; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_readlane_b32 s8, v47, 38
-; SI-NEXT: v_readlane_b32 s9, v47, 37
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: v_readlane_b32 s8, v47, 33
+; SI-NEXT: v_readlane_b32 s9, v47, 32
; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 8
; SI-NEXT: s_or_b32 s9, s8, s9
-; SI-NEXT: v_readlane_b32 s8, v47, 36
+; SI-NEXT: v_readlane_b32 s8, v47, 31
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v47, 35
+; SI-NEXT: v_readlane_b32 s10, v47, 30
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s10, s10, 24
-; SI-NEXT: s_or_b32 s88, s10, s8
-; SI-NEXT: v_readlane_b32 s8, v47, 40
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v47, 39
+; SI-NEXT: s_or_b32 s14, s10, s8
+; SI-NEXT: s_and_b32 s8, s12, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_lshl_b32 s10, s11, 24
; SI-NEXT: s_or_b32 s8, s10, s8
-; SI-NEXT: v_readlane_b32 s10, v47, 30
-; SI-NEXT: v_readlane_b32 s11, v47, 29
+; SI-NEXT: v_readlane_b32 s10, v47, 28
+; SI-NEXT: v_readlane_b32 s11, v47, 27
; SI-NEXT: s_and_b32 s10, s10, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 8
; SI-NEXT: s_or_b32 s11, s10, s11
-; SI-NEXT: v_readlane_b32 s10, v47, 28
+; SI-NEXT: v_readlane_b32 s10, v47, 26
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v47, 27
+; SI-NEXT: v_readlane_b32 s12, v47, 25
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s12, s12, 24
-; SI-NEXT: s_or_b32 s95, s12, s10
-; SI-NEXT: v_readlane_b32 s10, v47, 32
-; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v47, 31
+; SI-NEXT: s_or_b32 s91, s12, s10
+; SI-NEXT: s_and_b32 s10, s13, 0xff
+; SI-NEXT: v_readlane_b32 s12, v47, 29
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_or_b32 s10, s12, s10
@@ -172362,13 +173312,16 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: v_readlane_b32 s13, v47, 21
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 8
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s13, s12, s13
; SI-NEXT: v_readlane_b32 s12, v47, 20
+; SI-NEXT: v_writelane_b32 v46, s14, 52
+; SI-NEXT: s_or_b32 s9, s9, s14
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: v_readlane_b32 s14, v47, 19
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_or_b32 s89, s14, s12
+; SI-NEXT: s_or_b32 s93, s14, s12
; SI-NEXT: v_readlane_b32 s12, v47, 24
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: v_readlane_b32 s14, v47, 23
@@ -172376,366 +173329,337 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_lshl_b32 s14, s14, 24
; SI-NEXT: s_or_b32 s12, s14, s12
; SI-NEXT: v_readlane_b32 s14, v47, 14
-; SI-NEXT: v_readlane_b32 s15, v47, 53
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: v_writelane_b32 v46, s71, 48
; SI-NEXT: s_or_b32 s15, s14, s15
-; SI-NEXT: v_readlane_b32 s14, v47, 52
-; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_writelane_b32 v46, s16, 49
-; SI-NEXT: v_readlane_b32 s16, v47, 51
+; SI-NEXT: s_and_b32 s14, s16, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: v_writelane_b32 v46, s18, 50
-; SI-NEXT: s_lshl_b32 s18, s16, 24
-; SI-NEXT: s_or_b32 s93, s18, s14
+; SI-NEXT: s_mov_b32 s67, s29
+; SI-NEXT: s_mov_b32 s29, s28
+; SI-NEXT: s_mov_b32 s28, s26
+; SI-NEXT: s_mov_b32 s26, s68
+; SI-NEXT: s_mov_b32 s68, s24
+; SI-NEXT: s_mov_b32 s24, s25
+; SI-NEXT: s_lshl_b32 s25, s17, 24
+; SI-NEXT: s_or_b32 s17, s25, s14
; SI-NEXT: v_readlane_b32 s14, v47, 16
+; SI-NEXT: v_writelane_b32 v46, s90, 53
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: v_readlane_b32 s16, v47, 15
+; SI-NEXT: v_writelane_b32 v46, s69, 54
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s18, s16, 24
-; SI-NEXT: v_readlane_b32 s16, v47, 49
-; SI-NEXT: s_or_b32 s14, s18, s14
-; SI-NEXT: s_and_b32 s18, s80, 0xff
-; SI-NEXT: s_lshl_b32 s40, s16, 8
-; SI-NEXT: v_readlane_b32 s16, v47, 48
-; SI-NEXT: s_or_b32 s41, s18, s40
-; SI-NEXT: s_and_b32 s18, s16, 0xff
-; SI-NEXT: v_readlane_b32 s16, v47, 47
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s40, s16, 24
-; SI-NEXT: s_mov_b32 s24, s21
-; SI-NEXT: s_or_b32 s21, s40, s18
-; SI-NEXT: s_and_b32 s40, s70, 0xff
-; SI-NEXT: v_readlane_b32 s18, v47, 50
-; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: s_lshl_b32 s42, s18, 24
-; SI-NEXT: v_readlane_b32 s18, v47, 46
-; SI-NEXT: s_or_b32 s40, s42, s40
-; SI-NEXT: s_and_b32 s42, s96, 0xff
-; SI-NEXT: s_lshl_b32 s43, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v47, 45
-; SI-NEXT: s_or_b32 s43, s42, s43
-; SI-NEXT: s_and_b32 s42, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v47, 44
-; SI-NEXT: s_mov_b32 s34, s76
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s76, s18, 24
-; SI-NEXT: s_or_b32 s69, s76, s42
-; SI-NEXT: s_and_b32 s42, s90, 0xff
+; SI-NEXT: s_lshl_b32 s25, s16, 24
+; SI-NEXT: s_or_b32 s14, s25, s14
+; SI-NEXT: s_and_b32 s25, s41, 0xff
+; SI-NEXT: v_writelane_b32 v46, s27, 55
+; SI-NEXT: s_lshl_b32 s27, s40, 8
+; SI-NEXT: s_or_b32 s25, s25, s27
+; SI-NEXT: s_and_b32 s27, s18, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s18, s23, 24
+; SI-NEXT: s_or_b32 s18, s18, s27
+; SI-NEXT: s_and_b32 s27, s42, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s40, s43, 24
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_or_b32 s40, s40, s27
+; SI-NEXT: s_or_b32 s41, s25, s18
+; SI-NEXT: s_and_b32 s25, vcc_hi, 0xff
+; SI-NEXT: s_lshl_b32 s27, vcc_lo, 8
+; SI-NEXT: s_or_b32 s27, s25, s27
+; SI-NEXT: s_and_b32 s25, s77, 0xff
+; SI-NEXT: s_lshl_b32 s25, s25, 16
+; SI-NEXT: s_lshl_b32 s42, s78, 24
+; SI-NEXT: s_or_b32 s66, s42, s25
+; SI-NEXT: s_and_b32 s42, s30, 0xff
; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s76, s79, 24
-; SI-NEXT: s_or_b32 s42, s76, s42
-; SI-NEXT: s_and_b32 s76, s98, 0xff
-; SI-NEXT: s_lshl_b32 s77, s78, 8
-; SI-NEXT: v_writelane_b32 v46, s20, 51
+; SI-NEXT: s_lshl_b32 s43, s34, 24
+; SI-NEXT: s_and_b32 s27, s27, 0xffff
+; SI-NEXT: s_or_b32 s42, s43, s42
+; SI-NEXT: s_or_b32 s43, s27, s66
+; SI-NEXT: s_and_b32 s27, s38, 0xff
+; SI-NEXT: s_lshl_b32 s76, s37, 8
+; SI-NEXT: s_or_b32 s76, s27, s76
+; SI-NEXT: s_and_b32 s27, s35, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s77, s36, 24
+; SI-NEXT: s_or_b32 s90, s77, s27
+; SI-NEXT: s_and_b32 s77, s39, 0xff
+; SI-NEXT: s_lshl_b32 s77, s77, 16
+; SI-NEXT: s_lshl_b32 s78, s48, 24
+; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: s_or_b32 s34, s78, s77
+; SI-NEXT: s_or_b32 s35, s76, s90
+; SI-NEXT: s_and_b32 s76, s52, 0xff
+; SI-NEXT: s_lshl_b32 s77, s51, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s31, 0xff
-; SI-NEXT: v_writelane_b32 v46, s78, 52
+; SI-NEXT: s_and_b32 s77, s49, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s35, 24
-; SI-NEXT: s_mov_b32 s16, s70
-; SI-NEXT: s_or_b32 s70, s78, s77
-; SI-NEXT: s_and_b32 s77, s55, 0xff
+; SI-NEXT: s_lshl_b32 s78, s50, 24
+; SI-NEXT: s_or_b32 s69, s78, s77
+; SI-NEXT: s_and_b32 s77, s53, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s97, 24
+; SI-NEXT: s_lshl_b32 s78, s54, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_or_b32 vcc_lo, s78, s77
-; SI-NEXT: s_or_b32 vcc_hi, s76, s70
-; SI-NEXT: s_and_b32 s76, s22, 0xff
-; SI-NEXT: s_lshl_b32 s77, s65, 8
+; SI-NEXT: s_or_b32 s36, s78, s77
+; SI-NEXT: s_or_b32 s37, s76, s69
+; SI-NEXT: s_and_b32 s76, s55, 0xff
+; SI-NEXT: s_lshl_b32 s77, s98, 8
+; SI-NEXT: v_readlane_b32 s16, v47, 40
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s94, 0xff
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_writelane_b32 v46, s70, 56
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s67, 24
-; SI-NEXT: s_or_b32 s71, s78, s77
-; SI-NEXT: s_and_b32 s77, s92, 0xff
+; SI-NEXT: s_lshl_b32 s78, s24, 24
+; SI-NEXT: v_readlane_b32 s16, v46, 54
+; SI-NEXT: s_or_b32 s70, s78, s77
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v46, 53
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s30, 24
+; SI-NEXT: s_lshl_b32 s78, s16, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s38, s78, s77
-; SI-NEXT: s_or_b32 s39, s76, s71
-; SI-NEXT: s_and_b32 s76, s26, 0xff
-; SI-NEXT: s_lshl_b32 s77, s25, 8
+; SI-NEXT: s_or_b32 s39, s76, s70
+; SI-NEXT: s_and_b32 s76, s79, 0xff
+; SI-NEXT: s_lshl_b32 s77, s82, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s23, 0xff
+; SI-NEXT: s_and_b32 s77, s71, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
; SI-NEXT: s_lshl_b32 s78, s87, 24
-; SI-NEXT: v_writelane_b32 v46, s80, 53
-; SI-NEXT: s_or_b32 s80, s78, s77
-; SI-NEXT: s_and_b32 s77, s36, 0xff
+; SI-NEXT: s_or_b32 s71, s78, s77
+; SI-NEXT: s_and_b32 s77, s47, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s48, 24
+; SI-NEXT: s_lshl_b32 s78, s56, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s48, s78, s77
-; SI-NEXT: s_or_b32 s49, s76, s80
-; SI-NEXT: s_and_b32 s76, s85, 0xff
-; SI-NEXT: s_lshl_b32 s77, s64, 8
-; SI-NEXT: v_writelane_b32 v46, s98, 54
+; SI-NEXT: s_or_b32 s49, s76, s71
+; SI-NEXT: s_and_b32 s76, s46, 0xff
+; SI-NEXT: s_lshl_b32 s77, s65, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s50, 0xff
-; SI-NEXT: v_writelane_b32 v46, s24, 55
+; SI-NEXT: s_and_b32 s77, s80, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s51, 24
-; SI-NEXT: v_writelane_b32 v46, s34, 56
-; SI-NEXT: v_writelane_b32 v46, s81, 57
-; SI-NEXT: s_or_b32 s81, s78, s77
-; SI-NEXT: s_and_b32 s77, s57, 0xff
+; SI-NEXT: s_lshl_b32 s78, s81, 24
+; SI-NEXT: s_or_b32 vcc_hi, s78, s77
+; SI-NEXT: s_and_b32 s77, s86, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s58, 24
+; SI-NEXT: s_lshl_b32 s78, s83, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 39
; SI-NEXT: s_or_b32 s50, s78, s77
-; SI-NEXT: s_or_b32 s51, s76, s81
-; SI-NEXT: s_and_b32 s76, s56, 0xff
-; SI-NEXT: s_lshl_b32 s77, s45, 8
-; SI-NEXT: v_writelane_b32 v46, s22, 58
+; SI-NEXT: s_or_b32 s51, s76, vcc_hi
+; SI-NEXT: s_and_b32 s76, s31, 0xff
+; SI-NEXT: s_lshl_b32 s77, s16, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s53, 0xff
-; SI-NEXT: v_writelane_b32 v46, s65, 59
+; SI-NEXT: s_and_b32 s77, s29, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s52, 24
-; SI-NEXT: v_writelane_b32 v46, s67, 60
-; SI-NEXT: v_writelane_b32 v46, s94, 61
-; SI-NEXT: s_mov_b32 s94, s82
-; SI-NEXT: s_or_b32 s82, s78, s77
-; SI-NEXT: s_and_b32 s77, s62, 0xff
+; SI-NEXT: s_lshl_b32 s78, s28, 24
+; SI-NEXT: s_or_b32 s31, s78, s77
+; SI-NEXT: s_and_b32 s77, s84, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s60, 24
+; SI-NEXT: s_lshl_b32 s78, s57, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s52, s78, s77
-; SI-NEXT: s_or_b32 s53, s76, s82
-; SI-NEXT: s_and_b32 s76, s44, 0xff
-; SI-NEXT: s_lshl_b32 s77, s54, 8
+; SI-NEXT: s_or_b32 s53, s76, s31
+; SI-NEXT: s_and_b32 s76, s67, 0xff
+; SI-NEXT: s_lshl_b32 s77, s26, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s28, 0xff
+; SI-NEXT: s_and_b32 s77, s68, 0xff
+; SI-NEXT: v_readlane_b32 s16, v47, 38
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s27, 24
-; SI-NEXT: v_writelane_b32 v46, s97, 62
-; SI-NEXT: s_mov_b32 s92, s28
-; SI-NEXT: s_mov_b32 s28, s27
-; SI-NEXT: s_mov_b32 s27, s26
-; SI-NEXT: s_mov_b32 s26, s25
-; SI-NEXT: s_mov_b32 s25, s87
-; SI-NEXT: s_or_b32 s87, s78, s77
-; SI-NEXT: s_and_b32 s77, s61, 0xff
-; SI-NEXT: s_mov_b32 s24, s23
-; SI-NEXT: s_mov_b32 s23, s16
+; SI-NEXT: s_lshl_b32 s78, s16, 24
+; SI-NEXT: s_or_b32 s82, s78, s77
+; SI-NEXT: s_and_b32 s77, s45, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s59, 24
+; SI-NEXT: s_lshl_b32 s78, s44, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: v_readlane_b32 s16, v46, 45
-; SI-NEXT: s_mov_b32 s85, s55
+; SI-NEXT: v_readlane_b32 s16, v47, 37
; SI-NEXT: s_or_b32 s54, s78, s77
-; SI-NEXT: s_or_b32 s55, s76, s87
-; SI-NEXT: s_and_b32 s76, s29, 0xff
+; SI-NEXT: s_or_b32 s55, s76, s82
+; SI-NEXT: s_and_b32 s76, s19, 0xff
; SI-NEXT: s_lshl_b32 s77, s16, 8
-; SI-NEXT: v_readlane_b32 s18, v46, 26
+; SI-NEXT: v_readlane_b32 s16, v47, 36
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s19, 0xff
-; SI-NEXT: v_readlane_b32 s16, v46, 49
-; SI-NEXT: s_and_b32 s44, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 28
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v47, 35
+; SI-NEXT: s_or_b32 s15, s15, s17
; SI-NEXT: s_lshl_b32 s77, s77, 16
; SI-NEXT: s_lshl_b32 s78, s16, 24
-; SI-NEXT: s_and_b32 s45, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 30
-; SI-NEXT: s_mov_b32 s20, s19
-; SI-NEXT: s_mov_b32 s19, s37
-; SI-NEXT: s_mov_b32 s37, s35
-; SI-NEXT: s_mov_b32 s35, s31
-; SI-NEXT: s_mov_b32 s31, s79
-; SI-NEXT: s_mov_b32 s79, s90
-; SI-NEXT: s_mov_b32 s90, s96
-; SI-NEXT: s_or_b32 s96, s78, s77
-; SI-NEXT: s_lshl_b32 s78, s46, 24
-; SI-NEXT: s_and_b32 s46, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 32
-; SI-NEXT: s_and_b32 s77, s47, 0xff
-; SI-NEXT: s_and_b32 s47, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 34
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_and_b32 s56, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 36
-; SI-NEXT: s_or_b32 s64, s78, s77
-; SI-NEXT: s_or_b32 s65, s76, s96
-; SI-NEXT: s_and_b32 s76, s17, 0xff
-; SI-NEXT: s_lshl_b32 s77, s99, 8
-; SI-NEXT: s_and_b32 s57, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 38
-; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s86, 0xff
-; SI-NEXT: s_and_b32 s30, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 40
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s83, 24
-; SI-NEXT: s_and_b32 s34, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 42
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_mov_b32 s16, s83
-; SI-NEXT: s_or_b32 s83, s78, s77
-; SI-NEXT: s_and_b32 s77, s84, 0xff
-; SI-NEXT: s_and_b32 s36, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 46
-; SI-NEXT: s_or_b32 s5, s5, s68
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s66, 24
-; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_and_b32 s84, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 47
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s66, s78, s77
-; SI-NEXT: s_or_b32 s67, s76, s83
-; SI-NEXT: s_mov_b32 s78, s86
-; SI-NEXT: s_and_b32 s86, s18, 0xffff
-; SI-NEXT: s_mov_b32 s76, s99
-; SI-NEXT: s_and_b32 s99, s74, 0xffff
-; SI-NEXT: s_and_b32 s18, s75, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v46, 49
+; SI-NEXT: s_and_b32 s44, s20, 0xffff
+; SI-NEXT: s_mov_b32 s25, s99
+; SI-NEXT: s_or_b32 s87, s78, s77
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: s_and_b32 s57, s22, 0xffff
+; SI-NEXT: s_and_b32 s22, s74, 0xffff
+; SI-NEXT: s_and_b32 s99, s75, 0xffff
; SI-NEXT: s_or_b32 s74, s44, s4
; SI-NEXT: s_mov_b32 s75, s5
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_or_b32 s7, s7, s91
-; SI-NEXT: v_writelane_b32 v46, s4, 26
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 27
-; SI-NEXT: s_lshr_b64 s[4:5], s[6:7], 16
-; SI-NEXT: s_or_b32 s9, s9, s88
-; SI-NEXT: v_writelane_b32 v46, s4, 28
+; SI-NEXT: s_lshr_b64 s[20:21], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[14:15], 16
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 29
-; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 16
-; SI-NEXT: s_or_b32 s11, s11, s95
-; SI-NEXT: v_writelane_b32 v46, s4, 30
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 31
-; SI-NEXT: s_lshr_b64 s[4:5], s[10:11], 16
-; SI-NEXT: s_or_b32 s13, s13, s89
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 33
-; SI-NEXT: s_lshr_b64 s[4:5], s[12:13], 16
-; SI-NEXT: s_or_b32 s15, s15, s93
-; SI-NEXT: v_writelane_b32 v46, s4, 34
-; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 35
-; SI-NEXT: s_lshr_b64 s[4:5], s[14:15], 16
-; SI-NEXT: s_or_b32 s41, s41, s21
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 37
-; SI-NEXT: s_lshr_b64 s[4:5], s[40:41], 16
-; SI-NEXT: s_or_b32 s43, s43, s69
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: v_writelane_b32 v46, s5, 39
-; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: v_writelane_b32 v46, s5, 41
-; SI-NEXT: s_lshr_b64 s[4:5], vcc, 16
-; SI-NEXT: s_and_b32 s97, s63, 0xffff
-; SI-NEXT: s_and_b32 s98, s72, 0xffff
-; SI-NEXT: s_and_b32 s22, s73, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s4, 42
+; SI-NEXT: s_lshl_b32 s77, s77, 16
+; SI-NEXT: s_lshl_b32 s78, s64, 24
+; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: v_readlane_b32 s5, v46, 50
+; SI-NEXT: s_or_b32 s11, s11, s91
+; SI-NEXT: s_or_b32 s13, s13, s93
+; SI-NEXT: s_mov_b32 s27, s98
+; SI-NEXT: s_or_b32 s64, s78, s77
+; SI-NEXT: s_or_b32 s65, s76, s87
+; SI-NEXT: s_and_b32 s45, s88, 0xffff
+; SI-NEXT: s_and_b32 s46, s94, 0xffff
+; SI-NEXT: s_and_b32 s47, s85, 0xffff
+; SI-NEXT: s_and_b32 s56, s96, 0xffff
+; SI-NEXT: s_and_b32 vcc_lo, s58, 0xffff
+; SI-NEXT: s_and_b32 s30, s59, 0xffff
+; SI-NEXT: s_and_b32 s96, s60, 0xffff
+; SI-NEXT: s_and_b32 s83, s61, 0xffff
+; SI-NEXT: s_and_b32 s16, s62, 0xffff
+; SI-NEXT: s_and_b32 s86, s63, 0xffff
+; SI-NEXT: s_and_b32 s97, s72, 0xffff
+; SI-NEXT: s_and_b32 s98, s73, 0xffff
+; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 16
+; SI-NEXT: s_lshr_b32 s23, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v46, 51
; SI-NEXT: s_or_b32 s72, s45, s6
; SI-NEXT: s_mov_b32 s73, s7
; SI-NEXT: s_or_b32 s62, s46, s8
; SI-NEXT: s_mov_b32 s63, s9
+; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
; SI-NEXT: s_or_b32 s60, s47, s10
; SI-NEXT: s_mov_b32 s61, s11
+; SI-NEXT: s_lshr_b64 s[80:81], s[10:11], 16
; SI-NEXT: s_or_b32 s58, s56, s12
; SI-NEXT: s_mov_b32 s59, s13
+; SI-NEXT: s_lshr_b64 s[10:11], s[12:13], 16
; SI-NEXT: s_or_b32 s56, s57, s14
; SI-NEXT: s_mov_b32 s57, s15
-; SI-NEXT: s_or_b32 s46, s30, s40
+; SI-NEXT: s_or_b32 s46, vcc_lo, s40
; SI-NEXT: s_mov_b32 s47, s41
-; SI-NEXT: v_writelane_b32 v46, s5, 43
-; SI-NEXT: s_or_b32 s40, s84, s38
-; SI-NEXT: s_mov_b32 s41, s39
+; SI-NEXT: s_lshr_b64 s[6:7], s[40:41], 16
+; SI-NEXT: s_or_b32 s44, s30, s42
+; SI-NEXT: s_mov_b32 s45, s43
+; SI-NEXT: s_lshr_b64 s[88:89], s[42:43], 16
+; SI-NEXT: s_or_b32 s42, s96, s34
+; SI-NEXT: s_mov_b32 s43, s35
+; SI-NEXT: s_lshr_b64 s[84:85], s[34:35], 16
+; SI-NEXT: s_or_b32 s40, s83, s36
+; SI-NEXT: s_mov_b32 s41, s37
+; SI-NEXT: s_lshr_b64 s[8:9], s[36:37], 16
+; SI-NEXT: s_or_b32 s14, s16, s38
+; SI-NEXT: s_mov_b32 s15, s39
; SI-NEXT: s_lshr_b64 s[38:39], s[38:39], 16
-; SI-NEXT: s_or_b32 s14, s86, s48
-; SI-NEXT: s_mov_b32 s15, s49
+; SI-NEXT: s_or_b32 s12, s86, s48
+; SI-NEXT: s_mov_b32 s13, s49
; SI-NEXT: s_lshr_b64 s[48:49], s[48:49], 16
-; SI-NEXT: s_or_b32 s12, s97, s50
-; SI-NEXT: s_mov_b32 s13, s51
+; SI-NEXT: s_or_b32 s94, s97, s50
+; SI-NEXT: s_mov_b32 s95, s51
; SI-NEXT: s_lshr_b64 s[50:51], s[50:51], 16
-; SI-NEXT: s_or_b32 s10, s98, s52
-; SI-NEXT: s_mov_b32 s11, s53
+; SI-NEXT: s_or_b32 s36, s98, s52
+; SI-NEXT: s_mov_b32 s37, s53
; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
-; SI-NEXT: s_or_b32 s8, s22, s54
-; SI-NEXT: s_mov_b32 s9, s55
+; SI-NEXT: s_or_b32 s34, s22, s54
+; SI-NEXT: s_mov_b32 s35, s55
; SI-NEXT: s_lshr_b64 s[54:55], s[54:55], 16
-; SI-NEXT: s_or_b32 s6, s99, s64
-; SI-NEXT: s_mov_b32 s7, s65
+; SI-NEXT: s_lshr_b32 s77, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v46, 52
+; SI-NEXT: v_readlane_b32 s22, v46, 49
+; SI-NEXT: s_or_b32 s96, s99, s64
+; SI-NEXT: s_mov_b32 s97, s65
; SI-NEXT: s_lshr_b64 s[64:65], s[64:65], 16
-; SI-NEXT: s_or_b32 s4, s18, s66
-; SI-NEXT: s_mov_b32 s5, s67
-; SI-NEXT: s_lshr_b64 s[66:67], s[66:67], 16
-; SI-NEXT: s_or_b32 s44, s34, s42
-; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_or_b32 s42, s36, vcc_lo
-; SI-NEXT: s_mov_b32 s43, vcc_hi
-; SI-NEXT: s_mov_b32 s86, s78
-; SI-NEXT: s_mov_b32 s99, s76
-; SI-NEXT: v_readlane_b32 s18, v46, 50
-; SI-NEXT: s_lshr_b32 s77, s68, 16
-; SI-NEXT: s_lshr_b32 s68, s91, 16
-; SI-NEXT: s_lshr_b32 s88, s88, 16
-; SI-NEXT: s_lshr_b32 s84, s95, 16
-; SI-NEXT: s_lshr_b32 s89, s89, 16
-; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s93, s21, 16
-; SI-NEXT: s_lshr_b32 s95, s69, 16
-; SI-NEXT: s_lshr_b32 s30, s70, 16
-; SI-NEXT: s_lshr_b32 s34, s71, 16
-; SI-NEXT: s_lshr_b32 s36, s80, 16
-; SI-NEXT: v_readlane_b32 s69, v46, 44
-; SI-NEXT: v_readlane_b32 s80, v46, 53
-; SI-NEXT: s_lshr_b32 s39, s81, 16
-; SI-NEXT: v_readlane_b32 s81, v46, 57
-; SI-NEXT: v_readlane_b32 s76, v46, 56
-; SI-NEXT: v_readlane_b32 s21, v46, 55
-; SI-NEXT: v_readlane_b32 s98, v46, 54
-; SI-NEXT: s_lshr_b32 s49, s82, 16
-; SI-NEXT: s_mov_b32 s82, s94
-; SI-NEXT: v_readlane_b32 s71, v46, 48
-; SI-NEXT: v_readlane_b32 s78, v46, 52
-; SI-NEXT: s_mov_b32 s55, s85
-; SI-NEXT: v_readlane_b32 s97, v46, 62
-; SI-NEXT: v_readlane_b32 s94, v46, 61
-; SI-NEXT: v_readlane_b32 s67, v46, 60
-; SI-NEXT: v_readlane_b32 s65, v46, 59
-; SI-NEXT: v_readlane_b32 s22, v46, 58
-; SI-NEXT: s_lshr_b32 s51, s87, 16
-; SI-NEXT: s_mov_b32 s70, s23
-; SI-NEXT: s_mov_b32 s23, s24
-; SI-NEXT: v_readlane_b32 s24, v46, 45
-; SI-NEXT: s_mov_b32 s87, s25
-; SI-NEXT: s_mov_b32 s25, s26
-; SI-NEXT: s_mov_b32 s26, s27
-; SI-NEXT: s_mov_b32 s27, s28
-; SI-NEXT: s_mov_b32 s28, s92
-; SI-NEXT: s_lshr_b32 s53, s96, 16
-; SI-NEXT: s_mov_b32 s96, s90
-; SI-NEXT: s_mov_b32 s90, s79
-; SI-NEXT: s_mov_b32 s79, s31
-; SI-NEXT: s_mov_b32 s31, s35
-; SI-NEXT: s_mov_b32 s35, s37
-; SI-NEXT: s_mov_b32 s37, s19
-; SI-NEXT: s_mov_b32 s19, s20
-; SI-NEXT: v_readlane_b32 s20, v46, 51
-; SI-NEXT: s_lshr_b32 s92, s83, 16
-; SI-NEXT: s_mov_b32 s83, s16
-; SI-NEXT: v_readlane_b32 s16, v46, 49
-; SI-NEXT: s_cbranch_execnz .LBB93_3
-; SI-NEXT: .LBB93_2: ; %cmp.true
-; SI-NEXT: v_readlane_b32 s4, v46, 11
-; SI-NEXT: s_add_i32 s85, s4, 3
-; SI-NEXT: v_readlane_b32 s5, v46, 6
-; SI-NEXT: v_readlane_b32 s6, v47, 60
+; SI-NEXT: s_lshr_b32 s5, s5, 16
+; SI-NEXT: s_lshr_b32 s91, s91, 16
+; SI-NEXT: s_lshr_b32 s93, s93, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s66, 16
+; SI-NEXT: s_lshr_b32 s7, s90, 16
+; SI-NEXT: s_lshr_b32 s9, s69, 16
+; SI-NEXT: s_lshr_b32 s39, s70, 16
+; SI-NEXT: s_mov_b32 s98, s27
+; SI-NEXT: s_mov_b32 s99, s25
+; SI-NEXT: v_readlane_b32 s70, v46, 56
+; SI-NEXT: v_readlane_b32 s27, v46, 55
+; SI-NEXT: s_lshr_b32 s49, s71, 16
+; SI-NEXT: s_lshr_b32 s51, vcc_hi, 16
+; SI-NEXT: s_mov_b32 s25, s24
+; SI-NEXT: s_mov_b32 s24, s68
+; SI-NEXT: s_mov_b32 s68, s26
+; SI-NEXT: s_mov_b32 s26, s28
+; SI-NEXT: s_mov_b32 s28, s29
+; SI-NEXT: s_mov_b32 s29, s67
+; SI-NEXT: v_readlane_b32 s69, v46, 54
+; SI-NEXT: v_readlane_b32 s90, v46, 53
+; SI-NEXT: s_lshr_b32 s53, s31, 16
+; SI-NEXT: s_lshr_b32 s55, s82, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s87, 16
+; SI-NEXT: s_mov_b64 s[66:67], 0
+; SI-NEXT: s_branch .LBB93_3
+; SI-NEXT: .LBB93_2:
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: .LBB93_3: ; %Flow
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s17, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s17, 1
+; SI-NEXT: s_cbranch_scc1 .LBB93_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s85, s99, 3
+; SI-NEXT: v_readlane_b32 s5, v46, 29
; SI-NEXT: s_and_b32 s4, s85, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: s_add_i32 s6, s22, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_readlane_b32 s5, v47, 58
+; SI-NEXT: v_readlane_b32 s5, v46, 20
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
@@ -172743,27 +173667,30 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s5, s17, 3
+; SI-NEXT: s_add_i32 s5, s19, 3
+; SI-NEXT: v_readlane_b32 s6, v47, 37
+; SI-NEXT: v_readlane_b32 s7, v47, 36
; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_lshl_b32 s6, s99, 8
-; SI-NEXT: s_add_i32 s7, s86, 3
+; SI-NEXT: s_lshl_b32 s6, s6, 8
+; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: v_readlane_b32 s6, v47, 35
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s83, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_addk_i32 s5, 0x300
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s6, v46, 20
+; SI-NEXT: v_readlane_b32 s6, v46, 42
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 14
-; SI-NEXT: v_readlane_b32 s8, v46, 9
+; SI-NEXT: v_readlane_b32 s7, v46, 36
+; SI-NEXT: v_readlane_b32 s8, v46, 32
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
; SI-NEXT: s_add_i32 s83, s8, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 8
+; SI-NEXT: v_readlane_b32 s7, v46, 31
; SI-NEXT: s_and_b32 s8, s83, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
@@ -172773,25 +173700,27 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s7, s29, 3
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s8, s24, 8
-; SI-NEXT: s_add_i32 s9, s19, 3
+; SI-NEXT: s_lshl_b32 s8, s68, 8
+; SI-NEXT: s_add_i32 s9, s24, 3
+; SI-NEXT: s_add_i32 s96, s4, 0x3000000
; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: v_readlane_b32 s4, v47, 38
; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: s_lshl_b32 s8, s16, 24
+; SI-NEXT: s_lshl_b32 s8, s4, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_addk_i32 s7, 0x300
; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: v_readlane_b32 s8, v46, 23
+; SI-NEXT: v_readlane_b32 s8, v46, 45
; SI-NEXT: s_add_i32 s8, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 22
-; SI-NEXT: v_readlane_b32 s10, v46, 19
+; SI-NEXT: v_readlane_b32 s9, v46, 44
+; SI-NEXT: v_readlane_b32 s10, v46, 41
; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 8
; SI-NEXT: s_add_i32 s68, s10, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readlane_b32 s9, v46, 16
+; SI-NEXT: v_readlane_b32 s9, v46, 38
; SI-NEXT: s_and_b32 s10, s68, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
@@ -172799,45 +173728,45 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s9, s9, s10
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readlane_b32 s9, v46, 4
+; SI-NEXT: v_readlane_b32 s9, v46, 48
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s10, v46, 0
+; SI-NEXT: v_readlane_b32 s4, v47, 39
; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
+; SI-NEXT: s_lshl_b32 s10, s4, 8
; SI-NEXT: s_add_i32 s11, s28, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s10, s27, 24
+; SI-NEXT: s_lshl_b32 s10, s26, 24
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: s_addk_i32 s9, 0x300
; SI-NEXT: s_or_b32 s10, s10, s11
; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: v_readlane_b32 s10, v46, 25
+; SI-NEXT: v_readlane_b32 s10, v46, 47
; SI-NEXT: s_add_i32 s17, s10, 3
-; SI-NEXT: v_readlane_b32 s11, v46, 24
-; SI-NEXT: v_readlane_b32 s12, v46, 21
+; SI-NEXT: v_readlane_b32 s11, v46, 46
+; SI-NEXT: v_readlane_b32 s12, v46, 43
; SI-NEXT: s_and_b32 s10, s17, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_add_i32 s23, s12, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v46, 18
-; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: v_readlane_b32 s11, v46, 40
+; SI-NEXT: s_and_b32 s12, s23, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_addk_i32 s10, 0x300
; SI-NEXT: s_or_b32 s11, s11, s12
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v46, 10
+; SI-NEXT: v_readlane_b32 s11, v46, 33
; SI-NEXT: s_add_i32 s84, s11, 3
-; SI-NEXT: v_readlane_b32 s12, v46, 7
-; SI-NEXT: v_readlane_b32 s13, v46, 1
+; SI-NEXT: v_readlane_b32 s12, v46, 30
+; SI-NEXT: v_readlane_b32 s13, v46, 25
; SI-NEXT: s_and_b32 s11, s84, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_add_i32 s13, s13, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v47, 59
+; SI-NEXT: v_readlane_b32 s12, v46, 21
; SI-NEXT: s_and_b32 s13, s13, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
@@ -172845,15 +173774,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v46, 17
+; SI-NEXT: v_readlane_b32 s12, v46, 39
; SI-NEXT: s_add_i32 s99, s12, 3
-; SI-NEXT: v_readlane_b32 s13, v46, 15
-; SI-NEXT: v_readlane_b32 s14, v46, 12
+; SI-NEXT: v_readlane_b32 s13, v46, 37
+; SI-NEXT: v_readlane_b32 s14, v46, 34
; SI-NEXT: s_and_b32 s12, s99, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 8
; SI-NEXT: s_add_i32 s86, s14, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: v_readlane_b32 s13, v46, 13
+; SI-NEXT: v_readlane_b32 s13, v46, 35
; SI-NEXT: s_and_b32 s14, s86, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_lshl_b32 s14, s14, 16
@@ -172861,15 +173790,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s13, s13, s14
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: v_readlane_b32 s13, v46, 5
+; SI-NEXT: v_readlane_b32 s13, v46, 28
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s14, v46, 3
-; SI-NEXT: v_readlane_b32 s15, v47, 62
+; SI-NEXT: v_readlane_b32 s14, v46, 27
+; SI-NEXT: v_readlane_b32 s15, v46, 23
; SI-NEXT: s_and_b32 s13, s13, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_add_i32 s15, s15, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: v_readlane_b32 s14, v47, 63
+; SI-NEXT: v_readlane_b32 s14, v46, 24
; SI-NEXT: s_and_b32 s15, s15, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 24
; SI-NEXT: s_lshl_b32 s15, s15, 16
@@ -172877,41 +173806,41 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s14, s14, s15
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: v_readlane_b32 s14, v46, 2
+; SI-NEXT: v_readlane_b32 s14, v46, 26
; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: v_readlane_b32 s15, v47, 61
-; SI-NEXT: v_readlane_b32 s16, v47, 56
+; SI-NEXT: v_readlane_b32 s15, v46, 22
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s16, s69, 3
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: v_readlane_b32 s15, v47, 57
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s15, s15, 24
+; SI-NEXT: s_lshl_b32 s15, s90, 24
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_addk_i32 s14, 0x300
; SI-NEXT: s_or_b32 s15, s15, s16
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_add_i32 s15, s26, 3
+; SI-NEXT: v_readlane_b32 s15, v46, 19
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: v_readlane_b32 s4, v47, 40
; SI-NEXT: s_and_b32 s15, s15, 0xff
-; SI-NEXT: s_lshl_b32 s16, s25, 8
-; SI-NEXT: s_add_i32 s17, s23, 3
+; SI-NEXT: s_lshl_b32 s16, s98, 8
+; SI-NEXT: s_add_i32 s17, s4, 3
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: s_lshl_b32 s16, s87, 24
+; SI-NEXT: s_lshl_b32 s16, s25, 24
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_addk_i32 s15, 0x300
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
-; SI-NEXT: s_add_i32 s16, s20, 3
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v47, 54
+; SI-NEXT: s_add_i32 s16, s27, 3
+; SI-NEXT: v_readlane_b32 s18, v46, 17
; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s70, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 55
+; SI-NEXT: v_readlane_b32 s17, v46, 18
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -172920,66 +173849,83 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s40, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s22, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 15
+; SI-NEXT: v_readlane_b32 s18, v46, 12
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s65, 8
-; SI-NEXT: s_add_i32 s18, s94, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 13
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s67, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s41, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s71, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 14
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 11
+; SI-NEXT: v_readlane_b32 s18, v46, 9
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s81, 8
-; SI-NEXT: s_add_i32 s18, s55, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 10
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s97, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s42, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s98, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 8
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 7
+; SI-NEXT: v_readlane_b32 s18, v46, 3
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s78, 8
-; SI-NEXT: s_add_i32 s18, s31, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 4
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s35, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s43, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s82, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 6
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 5
+; SI-NEXT: v_readlane_b32 s18, v46, 1
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s37, 8
-; SI-NEXT: s_add_i32 s18, s90, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 2
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s79, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s44, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s96, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 46
-; SI-NEXT: v_readlane_b32 s18, v47, 45
+; SI-NEXT: v_readlane_b32 s16, v46, 0
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 63
+; SI-NEXT: v_readlane_b32 s18, v47, 59
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 44
+; SI-NEXT: v_readlane_b32 s17, v47, 60
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -172988,12 +173934,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s45, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s21, 3
+; SI-NEXT: v_readlane_b32 s16, v47, 62
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 61
+; SI-NEXT: v_readlane_b32 s18, v47, 57
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s76, 8
-; SI-NEXT: s_add_i32 s18, s70, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 50
+; SI-NEXT: v_readlane_b32 s17, v47, 58
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -173002,14 +173951,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s46, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s80, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 49
-; SI-NEXT: v_readlane_b32 s18, v47, 48
+; SI-NEXT: v_readlane_b32 s16, v47, 56
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 55
+; SI-NEXT: v_readlane_b32 s18, v47, 53
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 47
+; SI-NEXT: v_readlane_b32 s17, v47, 54
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -173037,8 +173987,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_add_i32 s56, s16, 0x3000000
; SI-NEXT: v_readlane_b32 s16, v47, 14
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 53
-; SI-NEXT: v_readlane_b32 s18, v47, 52
+; SI-NEXT: v_readlane_b32 s17, v47, 52
+; SI-NEXT: v_readlane_b32 s18, v47, 50
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -173052,12 +174002,11 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s57, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 26
+; SI-NEXT: v_readlane_b32 s16, v47, 45
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 25
; SI-NEXT: v_readlane_b32 s18, v47, 24
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s92, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: v_readlane_b32 s17, v47, 23
@@ -173086,32 +174035,32 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s59, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 34
+; SI-NEXT: v_readlane_b32 s16, v47, 46
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 33
-; SI-NEXT: v_readlane_b32 s18, v47, 32
+; SI-NEXT: v_readlane_b32 s17, v47, 44
+; SI-NEXT: v_readlane_b32 s18, v47, 41
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 31
+; SI-NEXT: v_readlane_b32 s4, v47, 29
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s17, s4, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s60, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 30
+; SI-NEXT: v_readlane_b32 s16, v47, 28
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 29
-; SI-NEXT: v_readlane_b32 s18, v47, 28
+; SI-NEXT: v_readlane_b32 s17, v47, 27
+; SI-NEXT: v_readlane_b32 s18, v47, 26
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 27
+; SI-NEXT: v_readlane_b32 s17, v47, 25
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -173120,14 +174069,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s61, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s69, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 41
-; SI-NEXT: v_readlane_b32 s18, v47, 40
+; SI-NEXT: v_readlane_b32 s16, v47, 49
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 47
+; SI-NEXT: v_readlane_b32 s18, v47, 43
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 39
+; SI-NEXT: v_readlane_b32 s17, v47, 42
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -173136,15 +174086,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s62, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 38
+; SI-NEXT: v_readlane_b32 s16, v47, 33
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 37
-; SI-NEXT: v_readlane_b32 s18, v47, 36
+; SI-NEXT: v_readlane_b32 s17, v47, 32
+; SI-NEXT: v_readlane_b32 s18, v47, 31
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 35
+; SI-NEXT: v_readlane_b32 s17, v47, 30
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -173173,15 +174123,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: v_readlane_b32 s16, v47, 1
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: v_readlane_b32 s17, v47, 0
-; SI-NEXT: v_readlane_b32 s18, v47, 43
+; SI-NEXT: v_readlane_b32 s18, v47, 48
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 42
+; SI-NEXT: v_readlane_b32 s4, v47, 34
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s17, s4, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
@@ -173219,191 +174169,154 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s75, s16, 0x3000000
-; SI-NEXT: s_lshr_b64 s[16:17], s[74:75], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 26
-; SI-NEXT: v_writelane_b32 v46, s17, 27
-; SI-NEXT: s_lshr_b64 s[16:17], s[72:73], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 28
-; SI-NEXT: v_writelane_b32 v46, s17, 29
-; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 30
-; SI-NEXT: v_writelane_b32 v46, s17, 31
-; SI-NEXT: s_lshr_b64 s[16:17], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 32
-; SI-NEXT: v_writelane_b32 v46, s17, 33
-; SI-NEXT: s_lshr_b64 s[16:17], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 34
-; SI-NEXT: v_writelane_b32 v46, s17, 35
-; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 36
-; SI-NEXT: v_writelane_b32 v46, s17, 37
-; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 38
-; SI-NEXT: v_writelane_b32 v46, s17, 39
-; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 16
-; SI-NEXT: s_add_i32 s10, s10, 0x3000000
-; SI-NEXT: s_add_i32 s11, s11, 0x3000000
+; SI-NEXT: s_add_i32 s34, s6, 0x3000000
+; SI-NEXT: s_add_i32 s35, s7, 0x3000000
+; SI-NEXT: s_add_i32 s36, s8, 0x3000000
+; SI-NEXT: s_add_i32 s37, s9, 0x3000000
+; SI-NEXT: s_add_i32 s94, s10, 0x3000000
+; SI-NEXT: s_add_i32 s95, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
; SI-NEXT: s_add_i32 s14, s14, 0x3000000
; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: v_writelane_b32 v46, s16, 40
-; SI-NEXT: s_add_i32 s4, s4, 0x3000000
-; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: s_add_i32 s8, s8, 0x3000000
-; SI-NEXT: s_add_i32 s9, s9, 0x3000000
-; SI-NEXT: v_writelane_b32 v46, s17, 41
-; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[50:51], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[52:53], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 42
-; SI-NEXT: s_lshr_b64 s[54:55], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[64:65], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[66:67], s[4:5], 16
-; SI-NEXT: s_lshr_b32 s77, s75, 16
-; SI-NEXT: s_lshr_b32 s68, s73, 16
-; SI-NEXT: s_lshr_b32 s88, s63, 16
-; SI-NEXT: s_lshr_b32 s84, s61, 16
-; SI-NEXT: s_lshr_b32 s89, s59, 16
-; SI-NEXT: s_lshr_b32 s91, s57, 16
-; SI-NEXT: s_lshr_b32 s93, s47, 16
-; SI-NEXT: s_lshr_b32 s95, s45, 16
-; SI-NEXT: s_lshr_b32 s30, s43, 16
-; SI-NEXT: s_lshr_b32 s34, s41, 16
-; SI-NEXT: s_lshr_b32 s36, s15, 16
-; SI-NEXT: s_lshr_b32 s39, s13, 16
-; SI-NEXT: s_lshr_b32 s49, s11, 16
-; SI-NEXT: s_lshr_b32 s51, s9, 16
-; SI-NEXT: s_lshr_b32 s53, s7, 16
-; SI-NEXT: s_lshr_b32 s92, s5, 16
-; SI-NEXT: v_writelane_b32 v46, s17, 43
-; SI-NEXT: .LBB93_3: ; %end
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s97, s5, 0x3000000
+; SI-NEXT: s_add_i32 s75, s16, 0x3000000
+; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[50:51], s[94:95], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[36:37], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[34:35], 16
+; SI-NEXT: s_lshr_b64 s[20:21], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[84:85], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[96:97], 16
+; SI-NEXT: s_lshr_b32 s23, s75, 16
+; SI-NEXT: s_lshr_b32 s77, s73, 16
+; SI-NEXT: s_lshr_b32 s5, s63, 16
+; SI-NEXT: s_lshr_b32 s91, s61, 16
+; SI-NEXT: s_lshr_b32 s93, s59, 16
+; SI-NEXT: s_lshr_b32 s11, s57, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s47, 16
+; SI-NEXT: s_lshr_b32 s30, s45, 16
+; SI-NEXT: s_lshr_b32 s7, s43, 16
+; SI-NEXT: s_lshr_b32 s9, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s15, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s51, s95, 16
+; SI-NEXT: s_lshr_b32 s53, s37, 16
+; SI-NEXT: s_lshr_b32 s55, s35, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s97, 16
+; SI-NEXT: .LBB93_5: ; %end
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s18, v46, 26
; SI-NEXT: s_and_b32 s17, s74, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v46, 27
-; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_lshl_b32 s18, s20, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s77, 16
-; SI-NEXT: v_readlane_b32 s20, v46, 28
+; SI-NEXT: s_lshl_b32 s19, s23, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s72, 0xffff
-; SI-NEXT: v_readlane_b32 s21, v46, 29
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_lshl_b32 s20, s76, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s68, 16
-; SI-NEXT: v_readlane_b32 s22, v46, 30
+; SI-NEXT: s_lshl_b32 s21, s77, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s62, 0xffff
-; SI-NEXT: v_readlane_b32 s23, v46, 31
-; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_lshl_b32 s22, s78, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s63, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s88, 16
-; SI-NEXT: v_readlane_b32 s24, v46, 32
+; SI-NEXT: s_lshl_b32 s23, s5, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s60, 0xffff
-; SI-NEXT: v_readlane_b32 s25, v46, 33
-; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_lshl_b32 s24, s80, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s61, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s84, 16
-; SI-NEXT: v_readlane_b32 s26, v46, 34
+; SI-NEXT: s_lshl_b32 s25, s91, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s58, 0xffff
-; SI-NEXT: v_readlane_b32 s27, v46, 35
-; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s26, s10, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s59, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s89, 16
-; SI-NEXT: v_readlane_b32 s28, v46, 36
+; SI-NEXT: s_lshl_b32 s27, s93, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s56, 0xffff
-; SI-NEXT: v_readlane_b32 s29, v46, 37
-; SI-NEXT: s_lshl_b32 s28, s28, 16
+; SI-NEXT: s_lshl_b32 s28, s4, 16
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s91, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 38
+; SI-NEXT: s_lshl_b32 s29, s11, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s46, 0xffff
-; SI-NEXT: v_readlane_b32 s57, v46, 39
-; SI-NEXT: s_lshl_b32 s46, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s6, 16
; SI-NEXT: s_or_b32 s29, s29, s46
; SI-NEXT: s_and_b32 s46, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s93, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 40
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_or_b32 s46, s46, s47
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: v_readlane_b32 s57, v46, 41
-; SI-NEXT: s_lshl_b32 s47, s56, 16
+; SI-NEXT: s_lshl_b32 s47, s88, 16
; SI-NEXT: s_or_b32 s44, s44, s47
; SI-NEXT: s_and_b32 s45, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s95, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 42
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_or_b32 s45, s45, s47
; SI-NEXT: s_and_b32 s42, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s56, 16
+; SI-NEXT: s_lshl_b32 s47, s84, 16
; SI-NEXT: s_or_b32 s42, s42, s47
; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, s7, 16
; SI-NEXT: s_or_b32 s43, s43, s47
; SI-NEXT: s_and_b32 s40, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s38, 16
+; SI-NEXT: s_lshl_b32 s47, s8, 16
; SI-NEXT: s_or_b32 s40, s40, s47
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s9, 16
; SI-NEXT: s_or_b32 s41, s41, s47
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s48, 16
+; SI-NEXT: s_lshl_b32 s47, s38, 16
; SI-NEXT: s_or_b32 s14, s14, s47
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshl_b32 s47, s39, 16
; SI-NEXT: s_or_b32 s15, s15, s47
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s50, 16
+; SI-NEXT: s_lshl_b32 s47, s48, 16
; SI-NEXT: s_or_b32 s12, s12, s47
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s39, 16
+; SI-NEXT: s_lshl_b32 s47, s49, 16
; SI-NEXT: s_or_b32 s13, s13, s47
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s52, 16
+; SI-NEXT: s_and_b32 s10, s94, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s50, 16
; SI-NEXT: s_or_b32 s10, s10, s47
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s49, 16
+; SI-NEXT: s_and_b32 s11, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s51, 16
; SI-NEXT: s_or_b32 s11, s11, s47
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s54, 16
+; SI-NEXT: s_and_b32 s8, s36, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s52, 16
; SI-NEXT: s_or_b32 s8, s8, s47
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s51, 16
+; SI-NEXT: s_and_b32 s9, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s53, 16
; SI-NEXT: s_or_b32 s9, s9, s47
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s64, 16
+; SI-NEXT: s_and_b32 s6, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s54, 16
; SI-NEXT: s_or_b32 s6, s6, s47
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s53, 16
+; SI-NEXT: s_and_b32 s7, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s55, 16
; SI-NEXT: s_or_b32 s7, s7, s47
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s66, 16
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s92, 16
+; SI-NEXT: s_and_b32 s4, s96, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s64, 16
+; SI-NEXT: s_and_b32 s5, s97, 0xffff
+; SI-NEXT: s_lshl_b32 s16, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s47
; SI-NEXT: s_or_b32 s5, s5, s16
; SI-NEXT: v_readlane_b32 s30, v45, 34
-; SI-NEXT: v_readlane_b32 s57, v46, 43
; SI-NEXT: v_mov_b32_e32 v0, s17
; SI-NEXT: v_mov_b32_e32 v1, s18
; SI-NEXT: v_mov_b32_e32 v2, s19
@@ -173478,74 +174391,6 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB93_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 26
-; SI-NEXT: v_writelane_b32 v46, s5, 27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: v_writelane_b32 v46, s4, 28
-; SI-NEXT: v_writelane_b32 v46, s5, 29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 30
-; SI-NEXT: v_writelane_b32 v46, s5, 31
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: v_writelane_b32 v46, s5, 33
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 34
-; SI-NEXT: v_writelane_b32 v46, s5, 35
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: v_writelane_b32 v46, s5, 37
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: v_writelane_b32 v46, s5, 39
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: v_writelane_b32 v46, s5, 41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 42
-; SI-NEXT: v_writelane_b32 v46, s5, 43
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB93_2
;
; VI-LABEL: bitcast_v128i8_to_v64f16_scalar:
; VI: ; %bb.0:
@@ -173762,7 +174607,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; VI-NEXT: s_cbranch_scc0 .LBB93_4
+; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
@@ -173982,12 +174827,32 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v35, v43
; VI-NEXT: v_perm_b32 v43, v44, v43, s4
; VI-NEXT: v_or_b32_e32 v31, v43, v31
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: v_mov_b32_e32 v43, v39
; VI-NEXT: v_mov_b32_e32 v39, v34
; VI-NEXT: v_mov_b32_e32 v34, v52
; VI-NEXT: v_mov_b32_e32 v52, v45
-; VI-NEXT: s_cbranch_execnz .LBB93_3
-; VI-NEXT: .LBB93_2: ; %cmp.true
+; VI-NEXT: s_branch .LBB93_3
+; VI-NEXT: .LBB93_2:
+; VI-NEXT: v_mov_b32_e32 v43, v39
+; VI-NEXT: v_mov_b32_e32 v52, v34
+; VI-NEXT: v_mov_b32_e32 v39, v33
+; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v34, v53
+; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v62, v2
+; VI-NEXT: v_mov_b32_e32 v35, v1
+; VI-NEXT: v_mov_b32_e32 v55, v57
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT: .LBB93_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB93_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -174402,7 +175267,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: v_or_b32_sdwa v27, v46, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v18, vcc, 0x3000000, v18
; VI-NEXT: v_add_u32_e32 v27, vcc, 0x3000000, v27
-; VI-NEXT: .LBB93_3: ; %end
+; VI-NEXT: .LBB93_5: ; %end
; VI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
@@ -174421,20 +175286,6 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB93_4:
-; VI-NEXT: v_mov_b32_e32 v43, v39
-; VI-NEXT: v_mov_b32_e32 v52, v34
-; VI-NEXT: v_mov_b32_e32 v39, v33
-; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v34, v53
-; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v62, v2
-; VI-NEXT: v_mov_b32_e32 v35, v1
-; VI-NEXT: v_mov_b32_e32 v55, v57
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT: s_branch .LBB93_2
;
; GFX9-LABEL: bitcast_v128i8_to_v64f16_scalar:
; GFX9: ; %bb.0:
@@ -174926,8 +175777,10 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB93_3: ; %Flow
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB93_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
@@ -175479,7 +176332,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB93_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB93_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v86
@@ -175631,9 +176484,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB93_3
-; GFX11-TRUE16-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB93_3
+; GFX11-TRUE16-NEXT: .LBB93_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB93_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
@@ -175862,16 +176724,12 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
-; GFX11-TRUE16-NEXT: .LBB93_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB93_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB93_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB93_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v64f16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -175995,7 +176853,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB93_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB93_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s74, 0xff
@@ -176186,9 +177044,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v40
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB93_3
-; GFX11-FAKE16-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB93_3
+; GFX11-FAKE16-NEXT: .LBB93_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB93_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v182
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v180
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v178
@@ -176449,14 +177316,10 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v29, v150, 16, v29
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v30, v30, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v35
-; GFX11-FAKE16-NEXT: .LBB93_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB93_5: ; %end
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:320 ; 4-byte Folded Reload
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB93_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB93_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -181308,164 +182171,165 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s8, s22, 16
; SI-NEXT: v_writelane_b32 v37, s7, 3
; SI-NEXT: s_lshr_b32 s9, s24, 16
-; SI-NEXT: v_readfirstlane_b32 s34, v18
-; SI-NEXT: v_readfirstlane_b32 s83, v17
-; SI-NEXT: v_readfirstlane_b32 s70, v16
-; SI-NEXT: v_readfirstlane_b32 s80, v15
-; SI-NEXT: v_readfirstlane_b32 s66, v14
-; SI-NEXT: v_readfirstlane_b32 s68, v13
-; SI-NEXT: v_readfirstlane_b32 s54, v12
-; SI-NEXT: v_readfirstlane_b32 s64, v11
-; SI-NEXT: v_readfirstlane_b32 s51, v10
-; SI-NEXT: v_readfirstlane_b32 s53, v9
-; SI-NEXT: v_readfirstlane_b32 s39, v8
-; SI-NEXT: v_readfirstlane_b32 s49, v7
-; SI-NEXT: v_readfirstlane_b32 s35, v6
-; SI-NEXT: v_readfirstlane_b32 s37, v5
-; SI-NEXT: v_readfirstlane_b32 s94, v4
-; SI-NEXT: v_readfirstlane_b32 s31, v3
-; SI-NEXT: v_readfirstlane_b32 s91, v2
-; SI-NEXT: v_readfirstlane_b32 s93, v1
+; SI-NEXT: v_readfirstlane_b32 s30, v18
+; SI-NEXT: v_readfirstlane_b32 s71, v17
+; SI-NEXT: v_readfirstlane_b32 s66, v16
+; SI-NEXT: v_readfirstlane_b32 s68, v15
+; SI-NEXT: v_readfirstlane_b32 s54, v14
+; SI-NEXT: v_readfirstlane_b32 s64, v13
+; SI-NEXT: v_readfirstlane_b32 s50, v12
+; SI-NEXT: v_readfirstlane_b32 s52, v11
+; SI-NEXT: v_readfirstlane_b32 s39, v10
+; SI-NEXT: v_readfirstlane_b32 s49, v9
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v8
+; SI-NEXT: v_readfirstlane_b32 s37, v7
+; SI-NEXT: v_readfirstlane_b32 s31, v6
+; SI-NEXT: v_readfirstlane_b32 s35, v5
+; SI-NEXT: v_readfirstlane_b32 s92, v4
+; SI-NEXT: v_readfirstlane_b32 s95, v3
+; SI-NEXT: v_readfirstlane_b32 s89, v2
+; SI-NEXT: v_readfirstlane_b32 s91, v1
; SI-NEXT: v_writelane_b32 v37, s8, 4
-; SI-NEXT: s_lshr_b32 s90, s29, 16
+; SI-NEXT: s_lshr_b32 s88, s29, 16
; SI-NEXT: s_lshr_b32 s11, s28, 16
-; SI-NEXT: s_lshr_b32 s89, s27, 16
+; SI-NEXT: s_lshr_b32 s79, s27, 16
; SI-NEXT: s_lshr_b32 s10, s26, 16
-; SI-NEXT: s_lshr_b32 s88, s25, 16
-; SI-NEXT: s_lshr_b32 s79, s23, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s77, s19, 16
-; SI-NEXT: s_lshr_b32 s76, s17, 16
-; SI-NEXT: s_lshr_b32 s82, s34, 16
-; SI-NEXT: s_lshr_b32 s84, s83, 16
-; SI-NEXT: s_lshr_b32 s71, s70, 16
-; SI-NEXT: s_lshr_b32 s81, s80, 16
+; SI-NEXT: s_lshr_b32 s78, s25, 16
+; SI-NEXT: s_lshr_b32 s77, s23, 16
+; SI-NEXT: s_lshr_b32 s76, s21, 16
+; SI-NEXT: s_lshr_b32 s99, s19, 16
+; SI-NEXT: s_lshr_b32 s98, s17, 16
+; SI-NEXT: s_lshr_b32 s70, s30, 16
+; SI-NEXT: s_lshr_b32 s80, s71, 16
; SI-NEXT: s_lshr_b32 s67, s66, 16
-; SI-NEXT: s_lshr_b32 s38, s68, 16
+; SI-NEXT: s_lshr_b32 s82, s68, 16
; SI-NEXT: s_lshr_b32 s55, s54, 16
-; SI-NEXT: s_lshr_b32 s99, s64, 16
-; SI-NEXT: s_lshr_b32 s52, s51, 16
-; SI-NEXT: s_lshr_b32 s69, s53, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s64, 16
+; SI-NEXT: s_lshr_b32 s51, s50, 16
+; SI-NEXT: s_lshr_b32 s87, s52, 16
; SI-NEXT: s_lshr_b32 s48, s39, 16
-; SI-NEXT: s_lshr_b32 s98, s49, 16
-; SI-NEXT: s_lshr_b32 s36, s35, 16
-; SI-NEXT: s_lshr_b32 s65, s37, 16
-; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s85, s31, 16
-; SI-NEXT: s_lshr_b32 s92, s91, 16
-; SI-NEXT: s_lshr_b32 s30, s93, 16
+; SI-NEXT: s_lshr_b32 s65, s49, 16
+; SI-NEXT: s_lshr_b32 s36, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s86, s37, 16
+; SI-NEXT: s_lshr_b32 s34, s31, 16
+; SI-NEXT: s_lshr_b32 s53, s35, 16
+; SI-NEXT: s_lshr_b32 s93, s92, 16
+; SI-NEXT: s_lshr_b32 s81, s95, 16
+; SI-NEXT: s_lshr_b32 s90, s89, 16
+; SI-NEXT: s_lshr_b32 s94, s91, 16
; SI-NEXT: v_readfirstlane_b32 s4, v19
; SI-NEXT: v_writelane_b32 v37, s9, 5
-; SI-NEXT: s_mov_b32 s97, s16
+; SI-NEXT: s_mov_b32 s85, s16
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_writelane_b32 v37, s10, 6
; SI-NEXT: v_writelane_b32 v37, s11, 7
; SI-NEXT: ; implicit-def: $vgpr36 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr35 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB95_3
+; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s97, 0xffff
+; SI-NEXT: s_and_b32 s4, s85, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s74, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s98, 16
; SI-NEXT: s_or_b32 s75, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s72, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s77, 16
+; SI-NEXT: s_lshl_b32 s5, s99, 16
; SI-NEXT: s_or_b32 s73, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s62, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s63, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s58, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s79, 16
+; SI-NEXT: s_lshl_b32 s5, s77, 16
; SI-NEXT: s_or_b32 s59, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s56, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s57, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s60, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s89, 16
+; SI-NEXT: s_lshl_b32 s5, s79, 16
; SI-NEXT: s_or_b32 s61, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_lshl_b32 s5, s88, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_and_b32 s4, s93, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
-; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s91, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_or_b32 s44, s4, s5
+; SI-NEXT: s_and_b32 s4, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s42, s4, s5
-; SI-NEXT: s_and_b32 s4, s94, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s95, 16
+; SI-NEXT: s_and_b32 s4, s92, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_or_b32 s43, s4, s5
-; SI-NEXT: s_and_b32 s4, s37, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
-; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: s_lshl_b32 s5, s53, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s41, s4, s5
-; SI-NEXT: s_and_b32 s4, s49, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_and_b32 s4, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s14, s4, s5
-; SI-NEXT: s_and_b32 s4, s39, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s48, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s36, 16
; SI-NEXT: s_or_b32 s15, s4, s5
-; SI-NEXT: s_and_b32 s4, s53, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s49, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s65, 16
; SI-NEXT: s_or_b32 s12, s4, s5
-; SI-NEXT: s_and_b32 s4, s51, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s52, 16
+; SI-NEXT: s_and_b32 s4, s39, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s48, 16
; SI-NEXT: s_or_b32 s13, s4, s5
-; SI-NEXT: s_and_b32 s4, s64, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_and_b32 s4, s52, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s10, s4, s5
-; SI-NEXT: s_and_b32 s4, s54, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: s_and_b32 s4, s50, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s51, 16
; SI-NEXT: s_or_b32 s11, s4, s5
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s38, 16
+; SI-NEXT: s_and_b32 s4, s64, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s8, s4, s5
-; SI-NEXT: s_and_b32 s4, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_and_b32 s4, s54, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s55, 16
; SI-NEXT: s_lshr_b32 s16, s75, 8
; SI-NEXT: s_or_b32 s9, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: v_writelane_b32 v35, s16, 2
; SI-NEXT: s_lshr_b32 s16, s73, 8
; SI-NEXT: s_or_b32 s6, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: v_writelane_b32 v35, s16, 1
; SI-NEXT: s_lshr_b32 s16, s63, 8
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: v_writelane_b32 v35, s16, 0
; SI-NEXT: s_lshr_b32 s16, s59, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s34, 0xffff
-; SI-NEXT: s_lshl_b32 vcc_lo, s82, 16
+; SI-NEXT: s_and_b32 s5, s30, 0xffff
+; SI-NEXT: s_mov_b32 s69, vcc_lo
+; SI-NEXT: s_lshl_b32 vcc_lo, s70, 16
; SI-NEXT: v_writelane_b32 v36, s16, 63
; SI-NEXT: s_lshr_b32 s16, s57, 8
; SI-NEXT: s_or_b32 s5, s5, vcc_lo
@@ -181492,43 +182356,41 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v36, vcc_lo, 52
; SI-NEXT: s_lshr_b32 vcc_lo, s5, 8
; SI-NEXT: v_writelane_b32 v36, vcc_lo, 51
-; SI-NEXT: s_bfe_u32 s16, s76, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s98, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 50
-; SI-NEXT: s_bfe_u32 s16, s77, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s99, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 49
-; SI-NEXT: s_bfe_u32 s16, s86, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s76, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 48
-; SI-NEXT: s_bfe_u32 s16, s79, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s77, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 47
-; SI-NEXT: s_bfe_u32 s16, s88, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s78, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 46
-; SI-NEXT: s_bfe_u32 s16, s89, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s79, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 45
-; SI-NEXT: s_bfe_u32 s16, s90, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s88, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 44
-; SI-NEXT: s_bfe_u32 s16, s92, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s90, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 43
-; SI-NEXT: s_bfe_u32 s16, s95, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s93, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 42
-; SI-NEXT: s_bfe_u32 s16, s36, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s34, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 41
-; SI-NEXT: s_bfe_u32 s16, s48, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s36, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 40
-; SI-NEXT: s_bfe_u32 s16, s52, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s48, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 39
-; SI-NEXT: s_bfe_u32 s16, s55, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s51, 0x80008
; SI-NEXT: v_writelane_b32 v36, s16, 38
-; SI-NEXT: s_mov_b32 s78, s81
-; SI-NEXT: s_mov_b32 s81, s38
-; SI-NEXT: s_mov_b32 s38, s77
-; SI-NEXT: s_mov_b32 s77, s99
-; SI-NEXT: s_mov_b32 s99, s69
-; SI-NEXT: s_mov_b32 s69, s76
-; SI-NEXT: s_mov_b32 s76, s98
-; SI-NEXT: s_mov_b32 s98, s65
-; SI-NEXT: s_mov_b32 s65, s97
-; SI-NEXT: s_mov_b32 s97, s85
-; SI-NEXT: s_mov_b32 s85, s17
+; SI-NEXT: s_mov_b32 vcc_lo, s99
+; SI-NEXT: s_mov_b32 s99, s87
+; SI-NEXT: s_mov_b32 s87, s65
+; SI-NEXT: s_mov_b32 s65, s98
+; SI-NEXT: s_mov_b32 s98, s86
+; SI-NEXT: s_mov_b32 s86, s53
+; SI-NEXT: s_mov_b32 s53, s85
+; SI-NEXT: s_mov_b32 s85, s81
+; SI-NEXT: s_mov_b32 s81, s17
; SI-NEXT: s_lshr_b64 s[16:17], s[74:75], 24
; SI-NEXT: v_writelane_b32 v37, s16, 12
; SI-NEXT: v_writelane_b32 v37, s17, 13
@@ -181547,24 +182409,24 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[16:17], s[72:73], 8
; SI-NEXT: v_writelane_b32 v37, s16, 14
; SI-NEXT: v_writelane_b32 v37, s17, 15
-; SI-NEXT: s_lshr_b64 vcc, s[62:63], 24
-; SI-NEXT: v_writelane_b32 v37, vcc_lo, 24
-; SI-NEXT: v_writelane_b32 v37, vcc_hi, 25
+; SI-NEXT: s_lshr_b64 s[96:97], s[62:63], 24
+; SI-NEXT: v_writelane_b32 v37, s96, 24
+; SI-NEXT: v_writelane_b32 v37, s97, 25
; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
; SI-NEXT: v_writelane_b32 v37, s16, 22
; SI-NEXT: v_writelane_b32 v37, s17, 23
; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 8
; SI-NEXT: v_writelane_b32 v37, s16, 20
; SI-NEXT: v_writelane_b32 v37, s17, 21
-; SI-NEXT: s_lshr_b64 vcc, s[58:59], 24
-; SI-NEXT: v_writelane_b32 v37, vcc_lo, 30
-; SI-NEXT: v_writelane_b32 v37, vcc_hi, 31
-; SI-NEXT: s_lshr_b64 vcc, s[58:59], 16
-; SI-NEXT: v_writelane_b32 v37, vcc_lo, 28
-; SI-NEXT: v_writelane_b32 v37, vcc_hi, 29
-; SI-NEXT: s_lshr_b64 vcc, s[58:59], 8
-; SI-NEXT: v_writelane_b32 v37, vcc_lo, 26
-; SI-NEXT: v_writelane_b32 v37, vcc_hi, 27
+; SI-NEXT: s_lshr_b64 s[96:97], s[58:59], 24
+; SI-NEXT: v_writelane_b32 v37, s96, 30
+; SI-NEXT: v_writelane_b32 v37, s97, 31
+; SI-NEXT: s_lshr_b64 s[96:97], s[58:59], 16
+; SI-NEXT: v_writelane_b32 v37, s96, 28
+; SI-NEXT: v_writelane_b32 v37, s97, 29
+; SI-NEXT: s_lshr_b64 s[96:97], s[58:59], 8
+; SI-NEXT: v_writelane_b32 v37, s96, 26
+; SI-NEXT: v_writelane_b32 v37, s97, 27
; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 24
; SI-NEXT: v_writelane_b32 v37, s16, 36
; SI-NEXT: v_writelane_b32 v37, s17, 37
@@ -181664,32 +182526,258 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[16:17], s[6:7], 8
; SI-NEXT: v_writelane_b32 v36, s16, 28
; SI-NEXT: v_writelane_b32 v36, s17, 29
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
-; SI-NEXT: v_writelane_b32 v36, vcc_lo, 36
-; SI-NEXT: v_writelane_b32 v36, vcc_hi, 37
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 8
+; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 16
+; SI-NEXT: v_writelane_b32 v36, s96, 36
+; SI-NEXT: v_writelane_b32 v36, s97, 37
+; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 8
; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v36, vcc_lo, 34
-; SI-NEXT: s_bfe_u32 s87, s67, 0x80008
-; SI-NEXT: s_bfe_u32 s50, s71, 0x80008
-; SI-NEXT: s_bfe_u32 s96, s82, 0x80008
-; SI-NEXT: s_mov_b32 s17, s85
-; SI-NEXT: s_mov_b32 s85, s97
-; SI-NEXT: s_mov_b32 s97, s65
-; SI-NEXT: s_mov_b32 s65, s98
-; SI-NEXT: s_mov_b32 s98, s76
-; SI-NEXT: s_mov_b32 s76, s69
-; SI-NEXT: s_mov_b32 s69, s99
-; SI-NEXT: s_mov_b32 s99, s77
-; SI-NEXT: s_mov_b32 s77, s38
-; SI-NEXT: s_mov_b32 s38, s81
-; SI-NEXT: s_mov_b32 s81, s78
-; SI-NEXT: v_writelane_b32 v36, vcc_hi, 35
-; SI-NEXT: s_cbranch_execnz .LBB95_4
-; SI-NEXT: .LBB95_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s83
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s84
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s82
+; SI-NEXT: v_writelane_b32 v36, s96, 34
+; SI-NEXT: s_bfe_u32 s83, s55, 0x80008
+; SI-NEXT: s_bfe_u32 s38, s67, 0x80008
+; SI-NEXT: s_bfe_u32 s84, s70, 0x80008
+; SI-NEXT: s_mov_b32 s17, s81
+; SI-NEXT: s_mov_b32 s81, s85
+; SI-NEXT: s_mov_b32 s85, s53
+; SI-NEXT: s_mov_b32 s53, s86
+; SI-NEXT: s_mov_b32 s86, s98
+; SI-NEXT: s_mov_b32 s98, s65
+; SI-NEXT: s_mov_b32 s65, s87
+; SI-NEXT: s_mov_b32 s87, s99
+; SI-NEXT: s_mov_b32 s99, vcc_lo
+; SI-NEXT: s_mov_b32 vcc_lo, s69
+; SI-NEXT: v_writelane_b32 v36, s97, 35
+; SI-NEXT: s_mov_b64 s[96:97], 0
+; SI-NEXT: s_branch .LBB95_3
+; SI-NEXT: .LBB95_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr16
+; SI-NEXT: s_mov_b64 s[96:97], -1
+; SI-NEXT: v_writelane_b32 v37, s4, 8
+; SI-NEXT: v_writelane_b32 v37, s5, 9
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: v_writelane_b32 v37, s4, 10
+; SI-NEXT: v_writelane_b32 v37, s5, 11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 12
+; SI-NEXT: v_writelane_b32 v37, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 14
+; SI-NEXT: v_writelane_b32 v37, s5, 15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 16
+; SI-NEXT: v_writelane_b32 v37, s5, 17
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 18
+; SI-NEXT: v_writelane_b32 v37, s5, 19
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 20
+; SI-NEXT: v_writelane_b32 v37, s5, 21
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 22
+; SI-NEXT: v_writelane_b32 v37, s5, 23
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 24
+; SI-NEXT: v_writelane_b32 v37, s5, 25
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 26
+; SI-NEXT: v_writelane_b32 v37, s5, 27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 28
+; SI-NEXT: v_writelane_b32 v37, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 30
+; SI-NEXT: v_writelane_b32 v37, s5, 31
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 32
+; SI-NEXT: v_writelane_b32 v37, s5, 33
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 34
+; SI-NEXT: v_writelane_b32 v37, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 36
+; SI-NEXT: v_writelane_b32 v37, s5, 37
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 38
+; SI-NEXT: v_writelane_b32 v37, s5, 39
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 40
+; SI-NEXT: v_writelane_b32 v37, s5, 41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 42
+; SI-NEXT: v_writelane_b32 v37, s5, 43
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 44
+; SI-NEXT: v_writelane_b32 v37, s5, 45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 46
+; SI-NEXT: v_writelane_b32 v37, s5, 47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 48
+; SI-NEXT: v_writelane_b32 v37, s5, 49
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 50
+; SI-NEXT: v_writelane_b32 v37, s5, 51
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 52
+; SI-NEXT: v_writelane_b32 v37, s5, 53
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 54
+; SI-NEXT: v_writelane_b32 v37, s5, 55
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 56
+; SI-NEXT: v_writelane_b32 v37, s5, 57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 58
+; SI-NEXT: v_writelane_b32 v37, s5, 59
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 60
+; SI-NEXT: v_writelane_b32 v37, s5, 61
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v37, s4, 62
+; SI-NEXT: v_writelane_b32 v37, s5, 63
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 0
+; SI-NEXT: v_writelane_b32 v36, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 2
+; SI-NEXT: v_writelane_b32 v36, s5, 3
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 4
+; SI-NEXT: v_writelane_b32 v36, s5, 5
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 6
+; SI-NEXT: v_writelane_b32 v36, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 8
+; SI-NEXT: v_writelane_b32 v36, s5, 9
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 10
+; SI-NEXT: v_writelane_b32 v36, s5, 11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 12
+; SI-NEXT: v_writelane_b32 v36, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 14
+; SI-NEXT: v_writelane_b32 v36, s5, 15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 16
+; SI-NEXT: v_writelane_b32 v36, s5, 17
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 18
+; SI-NEXT: v_writelane_b32 v36, s5, 19
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 20
+; SI-NEXT: v_writelane_b32 v36, s5, 21
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 22
+; SI-NEXT: v_writelane_b32 v36, s5, 23
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 24
+; SI-NEXT: v_writelane_b32 v36, s5, 25
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 26
+; SI-NEXT: v_writelane_b32 v36, s5, 27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 28
+; SI-NEXT: v_writelane_b32 v36, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 30
+; SI-NEXT: v_writelane_b32 v36, s5, 31
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s4, 32
+; SI-NEXT: v_writelane_b32 v36, s5, 33
+; SI-NEXT: v_writelane_b32 v36, s16, 34
+; SI-NEXT: v_writelane_b32 v36, s17, 35
+; SI-NEXT: ; implicit-def: $sgpr16
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v36, s16, 36
+; SI-NEXT: v_writelane_b32 v36, s17, 37
+; SI-NEXT: ; implicit-def: $sgpr16
+; SI-NEXT: .LBB95_3: ; %Flow
+; SI-NEXT: s_and_b64 s[96:97], s[96:97], exec
+; SI-NEXT: s_cselect_b32 s96, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s96, 1
+; SI-NEXT: s_cbranch_scc1 .LBB95_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
; SI-NEXT: v_readlane_b32 s46, v37, 7
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -181697,8 +182785,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v4, v1
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
; SI-NEXT: v_readfirstlane_b32 s5, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s82
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
@@ -181707,20 +182795,20 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_readfirstlane_b32 s6, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s67
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v2
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s66
; SI-NEXT: v_readfirstlane_b32 s7, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s38
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_lo
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
@@ -181730,17 +182818,17 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s7, v2
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s64
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: v_readfirstlane_b32 s8, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s67
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s55
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v5, v3
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s66
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s54
; SI-NEXT: v_readfirstlane_b32 s9, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s99
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s87
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -181748,19 +182836,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_readfirstlane_b32 s10, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s64
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s52
; SI-NEXT: v_readfirstlane_b32 s9, v3
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: v_readfirstlane_b32 s10, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s55
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s51
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v6, v4
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s54
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s50
; SI-NEXT: v_readfirstlane_b32 s11, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s65
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -181770,17 +182858,17 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s11, v4
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: v_readfirstlane_b32 s12, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s53
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s49
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: v_readfirstlane_b32 s12, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s52
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s48
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v7, v5
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s51
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s39
; SI-NEXT: v_readfirstlane_b32 s13, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s86
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -181788,19 +182876,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: v_readfirstlane_b32 s14, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s49
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s37
; SI-NEXT: v_readfirstlane_b32 s13, v5
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: v_readfirstlane_b32 s14, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s48
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s36
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v8, v6
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s39
+; SI-NEXT: v_cvt_f32_f16_e32 v7, vcc_hi
; SI-NEXT: v_readfirstlane_b32 s15, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s65
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s53
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -181810,17 +182898,17 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s15, v6
; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: v_readfirstlane_b32 s40, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s37
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s35
; SI-NEXT: s_or_b32 s15, s40, s15
; SI-NEXT: v_readfirstlane_b32 s40, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s36
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s34
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v9, v7
; SI-NEXT: s_lshl_b32 s40, s40, 16
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
; SI-NEXT: v_readfirstlane_b32 s41, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s81
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
@@ -181828,19 +182916,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: s_or_b32 s40, s41, s40
; SI-NEXT: v_readfirstlane_b32 s42, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s95
; SI-NEXT: v_readfirstlane_b32 s41, v7
; SI-NEXT: s_lshl_b32 s41, s41, 16
; SI-NEXT: s_or_b32 s41, s42, s41
; SI-NEXT: v_readfirstlane_b32 s42, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s95
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s93
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v10, v8
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s94
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s92
; SI-NEXT: v_readfirstlane_b32 s43, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s94
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -181850,15 +182938,15 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s43, v8
; SI-NEXT: s_lshl_b32 s43, s43, 16
; SI-NEXT: v_readfirstlane_b32 s44, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s93
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s91
; SI-NEXT: s_or_b32 s43, s44, s43
; SI-NEXT: v_readfirstlane_b32 s44, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s92
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s90
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v11, v9
; SI-NEXT: s_lshl_b32 s44, s44, 16
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s91
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s89
; SI-NEXT: v_readfirstlane_b32 s45, v11
; SI-NEXT: v_cvt_f32_f16_e32 v11, s46
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -181870,7 +182958,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s46, v10
; SI-NEXT: v_cvt_f32_f16_e32 v10, s28
; SI-NEXT: v_readfirstlane_b32 s28, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s90
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s88
; SI-NEXT: v_readfirstlane_b32 s45, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v12, v10
@@ -181890,7 +182978,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
; SI-NEXT: s_lshl_b32 s28, s28, 16
; SI-NEXT: v_readfirstlane_b32 s26, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s89
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s79
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v13, v11
; SI-NEXT: s_or_b32 s46, s46, s28
@@ -181922,7 +183010,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_readfirstlane_b32 s24, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s88
+; SI-NEXT: v_cvt_f32_f16_e32 v13, s78
; SI-NEXT: v_cvt_f32_f16_e32 v14, s25
; SI-NEXT: v_readfirstlane_b32 s25, v12
; SI-NEXT: v_cvt_f32_f16_e32 v12, s22
@@ -181940,7 +183028,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s24, v12
; SI-NEXT: v_cvt_f32_f16_e32 v12, s23
; SI-NEXT: v_readfirstlane_b32 s22, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s79
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s77
; SI-NEXT: v_readlane_b32 s23, v37, 3
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -181961,7 +183049,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s22, v12
; SI-NEXT: v_cvt_f32_f16_e32 v12, s21
; SI-NEXT: v_readfirstlane_b32 s20, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s76
; SI-NEXT: v_readlane_b32 s21, v37, 2
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -182001,7 +183089,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v36, s50, 14
; SI-NEXT: v_writelane_b32 v36, s51, 15
; SI-NEXT: s_lshr_b64 s[50:51], s[12:13], 16
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s77
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s99
; SI-NEXT: v_writelane_b32 v36, s50, 12
; SI-NEXT: v_writelane_b32 v36, s51, 13
; SI-NEXT: s_lshr_b64 s[50:51], s[12:13], 8
@@ -182021,7 +183109,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v36, s50, 16
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s85
; SI-NEXT: v_writelane_b32 v36, s51, 17
; SI-NEXT: s_lshr_b64 s[50:51], s[8:9], 24
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
@@ -182038,7 +183126,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_or_b32 s73, s16, s18
; SI-NEXT: v_readfirstlane_b32 s16, v14
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s76
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s98
; SI-NEXT: v_writelane_b32 v36, s50, 22
; SI-NEXT: v_cvt_f32_f16_e32 v15, s17
; SI-NEXT: v_writelane_b32 v36, s51, 23
@@ -182133,229 +183221,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b64 s[80:81], s[14:15], 8
; SI-NEXT: v_writelane_b32 v36, s51, 35
; SI-NEXT: s_mov_b32 s50, s86
-; SI-NEXT: s_branch .LBB95_5
-; SI-NEXT: .LBB95_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: v_writelane_b32 v37, s4, 8
-; SI-NEXT: v_writelane_b32 v37, s5, 9
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v37, s4, 10
-; SI-NEXT: v_writelane_b32 v37, s5, 11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 12
-; SI-NEXT: v_writelane_b32 v37, s5, 13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 14
-; SI-NEXT: v_writelane_b32 v37, s5, 15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 16
-; SI-NEXT: v_writelane_b32 v37, s5, 17
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 18
-; SI-NEXT: v_writelane_b32 v37, s5, 19
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 20
-; SI-NEXT: v_writelane_b32 v37, s5, 21
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 22
-; SI-NEXT: v_writelane_b32 v37, s5, 23
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 24
-; SI-NEXT: v_writelane_b32 v37, s5, 25
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 26
-; SI-NEXT: v_writelane_b32 v37, s5, 27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 28
-; SI-NEXT: v_writelane_b32 v37, s5, 29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 30
-; SI-NEXT: v_writelane_b32 v37, s5, 31
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 32
-; SI-NEXT: v_writelane_b32 v37, s5, 33
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 34
-; SI-NEXT: v_writelane_b32 v37, s5, 35
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 36
-; SI-NEXT: v_writelane_b32 v37, s5, 37
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 38
-; SI-NEXT: v_writelane_b32 v37, s5, 39
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 40
-; SI-NEXT: v_writelane_b32 v37, s5, 41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 42
-; SI-NEXT: v_writelane_b32 v37, s5, 43
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 44
-; SI-NEXT: v_writelane_b32 v37, s5, 45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 46
-; SI-NEXT: v_writelane_b32 v37, s5, 47
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 48
-; SI-NEXT: v_writelane_b32 v37, s5, 49
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 50
-; SI-NEXT: v_writelane_b32 v37, s5, 51
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 52
-; SI-NEXT: v_writelane_b32 v37, s5, 53
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 54
-; SI-NEXT: v_writelane_b32 v37, s5, 55
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 56
-; SI-NEXT: v_writelane_b32 v37, s5, 57
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 58
-; SI-NEXT: v_writelane_b32 v37, s5, 59
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 60
-; SI-NEXT: v_writelane_b32 v37, s5, 61
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v37, s4, 62
-; SI-NEXT: v_writelane_b32 v37, s5, 63
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 0
-; SI-NEXT: v_writelane_b32 v36, s5, 1
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 2
-; SI-NEXT: v_writelane_b32 v36, s5, 3
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 4
-; SI-NEXT: v_writelane_b32 v36, s5, 5
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 6
-; SI-NEXT: v_writelane_b32 v36, s5, 7
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 8
-; SI-NEXT: v_writelane_b32 v36, s5, 9
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 10
-; SI-NEXT: v_writelane_b32 v36, s5, 11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 12
-; SI-NEXT: v_writelane_b32 v36, s5, 13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 14
-; SI-NEXT: v_writelane_b32 v36, s5, 15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 16
-; SI-NEXT: v_writelane_b32 v36, s5, 17
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 18
-; SI-NEXT: v_writelane_b32 v36, s5, 19
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 20
-; SI-NEXT: v_writelane_b32 v36, s5, 21
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 22
-; SI-NEXT: v_writelane_b32 v36, s5, 23
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 24
-; SI-NEXT: v_writelane_b32 v36, s5, 25
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 26
-; SI-NEXT: v_writelane_b32 v36, s5, 27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 28
-; SI-NEXT: v_writelane_b32 v36, s5, 29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 30
-; SI-NEXT: v_writelane_b32 v36, s5, 31
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s4, 32
-; SI-NEXT: v_writelane_b32 v36, s5, 33
-; SI-NEXT: v_writelane_b32 v36, s16, 34
-; SI-NEXT: v_writelane_b32 v36, s17, 35
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v36, s16, 36
-; SI-NEXT: v_writelane_b32 v36, s17, 37
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: s_branch .LBB95_2
-; SI-NEXT: .LBB95_4:
+; SI-NEXT: s_branch .LBB95_6
+; SI-NEXT: .LBB95_5:
; SI-NEXT: v_readlane_b32 s17, v36, 38
; SI-NEXT: v_mov_b32_e32 v17, s17
; SI-NEXT: v_readlane_b32 s17, v36, 39
@@ -182381,12 +183248,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s17, v36, 49
; SI-NEXT: v_mov_b32_e32 v31, s17
; SI-NEXT: v_readlane_b32 s17, v36, 50
-; SI-NEXT: v_mov_b32_e32 v10, s90
-; SI-NEXT: v_mov_b32_e32 v11, s89
-; SI-NEXT: v_mov_b32_e32 v13, s88
-; SI-NEXT: v_mov_b32_e32 v16, s79
-; SI-NEXT: v_mov_b32_e32 v23, s77
-; SI-NEXT: v_mov_b32_e32 v27, s76
+; SI-NEXT: v_mov_b32_e32 v9, s90
+; SI-NEXT: v_mov_b32_e32 v10, s88
+; SI-NEXT: v_mov_b32_e32 v11, s79
+; SI-NEXT: v_mov_b32_e32 v13, s78
+; SI-NEXT: v_mov_b32_e32 v16, s77
+; SI-NEXT: v_mov_b32_e32 v20, s76
; SI-NEXT: v_mov_b32_e32 v32, s17
; SI-NEXT: v_readlane_b32 s17, v36, 51
; SI-NEXT: v_readlane_b32 s76, v37, 30
@@ -182399,18 +183266,17 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s91, v37, 27
; SI-NEXT: v_writelane_b32 v37, s16, 0
; SI-NEXT: v_writelane_b32 v37, s17, 1
-; SI-NEXT: v_mov_b32_e32 v8, s95
; SI-NEXT: v_readlane_b32 s94, v37, 22
; SI-NEXT: v_readlane_b32 s95, v37, 23
; SI-NEXT: s_mov_b32 s95, s17
; SI-NEXT: v_readlane_b32 s16, v36, 36
-; SI-NEXT: v_mov_b32_e32 v4, s55
-; SI-NEXT: v_mov_b32_e32 v6, s48
+; SI-NEXT: v_mov_b32_e32 v3, s55
+; SI-NEXT: v_mov_b32_e32 v5, s48
; SI-NEXT: v_readlane_b32 s17, v36, 37
; SI-NEXT: v_readlane_b32 s54, v37, 10
; SI-NEXT: v_readlane_b32 s48, v37, 42
-; SI-NEXT: v_mov_b32_e32 v5, s52
-; SI-NEXT: v_mov_b32_e32 v7, s36
+; SI-NEXT: v_mov_b32_e32 v6, s36
+; SI-NEXT: v_mov_b32_e32 v14, s38
; SI-NEXT: v_readlane_b32 s22, v36, 54
; SI-NEXT: v_readlane_b32 s23, v36, 57
; SI-NEXT: v_readlane_b32 s36, v37, 20
@@ -182429,12 +183295,15 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_mov_b32 s55, s23
; SI-NEXT: s_mov_b32 s49, s22
; SI-NEXT: v_readlane_b32 s22, v37, 38
-; SI-NEXT: v_mov_b32_e32 v1, s82
-; SI-NEXT: v_mov_b32_e32 v2, s71
-; SI-NEXT: v_mov_b32_e32 v3, s67
-; SI-NEXT: v_mov_b32_e32 v9, s92
-; SI-NEXT: v_mov_b32_e32 v12, s96
-; SI-NEXT: v_mov_b32_e32 v14, s50
+; SI-NEXT: v_mov_b32_e32 v1, s70
+; SI-NEXT: v_mov_b32_e32 v2, s67
+; SI-NEXT: v_mov_b32_e32 v4, s51
+; SI-NEXT: v_mov_b32_e32 v7, s34
+; SI-NEXT: v_mov_b32_e32 v8, s93
+; SI-NEXT: v_mov_b32_e32 v23, s99
+; SI-NEXT: v_mov_b32_e32 v27, s98
+; SI-NEXT: v_mov_b32_e32 v12, s84
+; SI-NEXT: v_mov_b32_e32 v15, s83
; SI-NEXT: v_readlane_b32 vcc_lo, v37, 18
; SI-NEXT: s_mov_b32 s37, s18
; SI-NEXT: s_mov_b32 s39, s20
@@ -182457,8 +183326,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s34, v37, 56
; SI-NEXT: v_readlane_b32 s70, v37, 52
; SI-NEXT: v_readlane_b32 s82, v37, 50
-; SI-NEXT: v_mov_b32_e32 v20, s86
-; SI-NEXT: v_mov_b32_e32 v15, s87
; SI-NEXT: v_readlane_b32 s19, v36, 56
; SI-NEXT: v_readlane_b32 s26, v36, 58
; SI-NEXT: v_readlane_b32 s27, v36, 59
@@ -182486,7 +183353,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s35, v37, 57
; SI-NEXT: v_readlane_b32 s71, v37, 53
; SI-NEXT: v_readlane_b32 s83, v37, 51
-; SI-NEXT: .LBB95_5: ; %end
+; SI-NEXT: .LBB95_6: ; %end
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_and_b32 s17, s74, 0xff
; SI-NEXT: s_or_b32 s16, s17, s16
@@ -183027,7 +183894,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
; VI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB95_3
+; VI-NEXT: s_cbranch_scc0 .LBB95_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s5, 24
; VI-NEXT: v_writelane_b32 v62, s46, 16
@@ -183178,13 +184045,175 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_lshr_b64 s[78:79], s[6:7], 24
; VI-NEXT: s_lshr_b64 s[88:89], s[8:9], 24
; VI-NEXT: s_lshr_b64 s[90:91], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[42:43], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[44:45], 24
-; VI-NEXT: s_cbranch_execnz .LBB95_4
-; VI-NEXT: .LBB95_2: ; %cmp.true
+; VI-NEXT: s_lshr_b64 vcc, s[12:13], 24
+; VI-NEXT: s_lshr_b64 s[30:31], s[14:15], 24
+; VI-NEXT: s_lshr_b64 s[34:35], s[40:41], 24
+; VI-NEXT: s_lshr_b64 s[36:37], s[42:43], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[44:45], 24
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB95_3
+; VI-NEXT: .LBB95_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB95_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s47, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s47, 1
+; VI-NEXT: s_cbranch_scc1 .LBB95_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s46, s45, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x200
; VI-NEXT: v_add_f16_e32 v36, s46, v7
@@ -183483,164 +184512,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_bfe_u32 v15, v39, 8, 8
; VI-NEXT: v_bfe_u32 v22, v60, 8, 8
; VI-NEXT: v_bfe_u32 v30, v42, 8, 8
-; VI-NEXT: s_branch .LBB95_5
-; VI-NEXT: .LBB95_3:
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB95_2
-; VI-NEXT: .LBB95_4:
+; VI-NEXT: s_branch .LBB95_6
+; VI-NEXT: .LBB95_5:
; VI-NEXT: v_mov_b32_e32 v1, s44
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s45
@@ -183891,12 +184764,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v20, s76
; VI-NEXT: v_mov_b32_e32 v2, s88
; VI-NEXT: v_mov_b32_e32 v3, s90
-; VI-NEXT: v_mov_b32_e32 v4, s30
-; VI-NEXT: v_mov_b32_e32 v5, s34
-; VI-NEXT: v_mov_b32_e32 v8, s36
-; VI-NEXT: v_mov_b32_e32 v24, s38
-; VI-NEXT: v_mov_b32_e32 v14, s48
-; VI-NEXT: .LBB95_5: ; %end
+; VI-NEXT: v_mov_b32_e32 v4, vcc_lo
+; VI-NEXT: v_mov_b32_e32 v5, s30
+; VI-NEXT: v_mov_b32_e32 v8, s34
+; VI-NEXT: v_mov_b32_e32 v24, s36
+; VI-NEXT: v_mov_b32_e32 v14, s38
+; VI-NEXT: .LBB95_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v20, v23, v20, s4
; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
@@ -184339,7 +185212,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB95_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
; GFX9-NEXT: v_writelane_b32 v62, s46, 49
@@ -184484,11 +185357,165 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b64 s[90:91], s[10:11], 24
; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[44:45], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB95_4
-; GFX9-NEXT: .LBB95_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[40:41], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[42:43], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB95_3
+; GFX9-NEXT: .LBB95_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB95_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v15, 0x200
; GFX9-NEXT: v_pk_add_f16 v26, s5, v15 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v25, s4, v15 op_sel_hi:[1,0]
@@ -184714,156 +185741,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 8, v19
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; GFX9-NEXT: s_branch .LBB95_5
-; GFX9-NEXT: .LBB95_3:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB95_2
-; GFX9-NEXT: .LBB95_4:
+; GFX9-NEXT: s_branch .LBB95_6
+; GFX9-NEXT: .LBB95_5:
; GFX9-NEXT: v_mov_b32_e32 v15, s81
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s71
@@ -185129,19 +186008,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s30
+; GFX9-NEXT: v_mov_b32_e32 v22, vcc_lo
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s34
+; GFX9-NEXT: v_mov_b32_e32 v22, s30
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s36
+; GFX9-NEXT: v_mov_b32_e32 v22, s34
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-NEXT: .LBB95_5: ; %end
+; GFX9-NEXT: .LBB95_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
@@ -185598,74 +186477,74 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s100, 0
; GFX11-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB95_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s27, 24
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s5, 24
+; GFX11-NEXT: s_lshr_b32 s30, s5, 24
; GFX11-NEXT: v_writelane_b32 v77, s42, 7
; GFX11-NEXT: s_lshr_b32 s42, s27, 16
-; GFX11-NEXT: s_lshr_b32 s34, s5, 16
-; GFX11-NEXT: s_lshr_b32 s36, s5, 8
-; GFX11-NEXT: s_lshr_b32 s35, s4, 16
+; GFX11-NEXT: s_lshr_b32 s31, s5, 16
+; GFX11-NEXT: s_lshr_b32 s35, s5, 8
+; GFX11-NEXT: s_lshr_b32 s34, s4, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 6
; GFX11-NEXT: s_lshr_b32 s42, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s4, 8
-; GFX11-NEXT: s_lshr_b32 s38, s7, 24
-; GFX11-NEXT: s_lshr_b32 s39, s7, 16
+; GFX11-NEXT: s_lshr_b32 s36, s4, 8
+; GFX11-NEXT: s_lshr_b32 s37, s7, 24
+; GFX11-NEXT: s_lshr_b32 s38, s7, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 5
; GFX11-NEXT: s_lshr_b32 s42, s26, 16
-; GFX11-NEXT: s_lshr_b32 s49, s7, 8
-; GFX11-NEXT: s_lshr_b32 s48, s6, 16
-; GFX11-NEXT: s_lshr_b32 s50, s6, 8
+; GFX11-NEXT: s_lshr_b32 s48, s7, 8
+; GFX11-NEXT: s_lshr_b32 s39, s6, 16
+; GFX11-NEXT: s_lshr_b32 s49, s6, 8
; GFX11-NEXT: v_writelane_b32 v77, s42, 4
; GFX11-NEXT: s_lshr_b32 s42, s26, 8
-; GFX11-NEXT: s_lshr_b32 s51, s9, 24
-; GFX11-NEXT: s_lshr_b32 s52, s9, 16
-; GFX11-NEXT: s_lshr_b32 s54, s9, 8
+; GFX11-NEXT: s_lshr_b32 s50, s9, 24
+; GFX11-NEXT: s_lshr_b32 s51, s9, 16
+; GFX11-NEXT: s_lshr_b32 s53, s9, 8
; GFX11-NEXT: v_writelane_b32 v77, s42, 3
; GFX11-NEXT: s_lshr_b32 s42, s25, 24
-; GFX11-NEXT: s_lshr_b32 s53, s8, 16
-; GFX11-NEXT: s_lshr_b32 s55, s8, 8
-; GFX11-NEXT: s_lshr_b32 s64, s11, 24
+; GFX11-NEXT: s_lshr_b32 s52, s8, 16
+; GFX11-NEXT: s_lshr_b32 s54, s8, 8
+; GFX11-NEXT: s_lshr_b32 s55, s11, 24
; GFX11-NEXT: v_writelane_b32 v77, s42, 2
; GFX11-NEXT: s_lshr_b32 s42, s25, 16
-; GFX11-NEXT: s_lshr_b32 s65, s11, 16
-; GFX11-NEXT: s_lshr_b32 s67, s11, 8
-; GFX11-NEXT: s_lshr_b32 s66, s10, 16
+; GFX11-NEXT: s_lshr_b32 s64, s11, 16
+; GFX11-NEXT: s_lshr_b32 s66, s11, 8
+; GFX11-NEXT: s_lshr_b32 s65, s10, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 1
; GFX11-NEXT: s_lshr_b32 s42, s25, 8
-; GFX11-NEXT: s_lshr_b32 s68, s10, 8
-; GFX11-NEXT: s_lshr_b32 s69, s13, 24
-; GFX11-NEXT: s_lshr_b32 s70, s13, 16
+; GFX11-NEXT: s_lshr_b32 s67, s10, 8
+; GFX11-NEXT: s_lshr_b32 s68, s13, 24
+; GFX11-NEXT: s_lshr_b32 s69, s13, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 0
; GFX11-NEXT: s_lshr_b32 s42, s24, 16
-; GFX11-NEXT: s_lshr_b32 s80, s13, 8
+; GFX11-NEXT: s_lshr_b32 s71, s13, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 31
; GFX11-NEXT: s_lshr_b32 s42, s24, 8
-; GFX11-NEXT: s_lshr_b32 s71, s12, 16
-; GFX11-NEXT: s_lshr_b32 s81, s12, 8
-; GFX11-NEXT: s_lshr_b32 s82, s15, 24
+; GFX11-NEXT: s_lshr_b32 s70, s12, 16
+; GFX11-NEXT: s_lshr_b32 s80, s12, 8
+; GFX11-NEXT: s_lshr_b32 s81, s15, 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 30
; GFX11-NEXT: s_lshr_b32 s42, s23, 24
-; GFX11-NEXT: s_lshr_b32 s83, s15, 16
-; GFX11-NEXT: s_lshr_b32 s85, s15, 8
-; GFX11-NEXT: s_lshr_b32 s84, s14, 16
+; GFX11-NEXT: s_lshr_b32 s82, s15, 16
+; GFX11-NEXT: s_lshr_b32 s84, s15, 8
+; GFX11-NEXT: s_lshr_b32 s83, s14, 16
; GFX11-NEXT: v_writelane_b32 v76, s42, 29
; GFX11-NEXT: s_lshr_b32 s42, s23, 16
-; GFX11-NEXT: s_lshr_b32 s86, s14, 8
-; GFX11-NEXT: s_lshr_b32 s87, s41, 24
-; GFX11-NEXT: s_lshr_b32 s96, s41, 16
+; GFX11-NEXT: s_lshr_b32 s85, s14, 8
+; GFX11-NEXT: s_lshr_b32 s86, s41, 24
+; GFX11-NEXT: s_lshr_b32 s87, s41, 16
; GFX11-NEXT: v_writelane_b32 v76, s42, 28
; GFX11-NEXT: s_lshr_b32 s42, s23, 8
-; GFX11-NEXT: s_lshr_b32 s98, s41, 8
-; GFX11-NEXT: s_lshr_b32 s97, s40, 16
-; GFX11-NEXT: s_lshr_b32 s99, s40, 8
+; GFX11-NEXT: s_lshr_b32 s97, s41, 8
+; GFX11-NEXT: s_lshr_b32 s96, s40, 16
+; GFX11-NEXT: s_lshr_b32 s98, s40, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 27
; GFX11-NEXT: s_lshr_b32 s42, s22, 16
-; GFX11-NEXT: s_lshr_b32 s100, s29, 24
+; GFX11-NEXT: s_lshr_b32 s99, s29, 24
; GFX11-NEXT: s_lshr_b32 s101, s29, 16
; GFX11-NEXT: s_lshr_b32 s103, s29, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 26
@@ -185696,7 +186575,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_writelane_b32 v76, s42, 21
; GFX11-NEXT: s_lshr_b32 s42, s20, 8
; GFX11-NEXT: s_lshr_b64 s[94:95], s[40:41], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[28:29], 24
+; GFX11-NEXT: s_lshr_b64 vcc, s[28:29], 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 20
; GFX11-NEXT: s_lshr_b32 s42, s19, 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -185749,9 +186628,152 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s42, s0, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 0
; GFX11-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB95_3
+; GFX11-NEXT: .LBB95_2:
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: s_mov_b32 s100, -1
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr104
+; GFX11-NEXT: ; implicit-def: $sgpr102
+; GFX11-NEXT: ; implicit-def: $sgpr103
+; GFX11-NEXT: ; implicit-def: $sgpr101
+; GFX11-NEXT: ; implicit-def: $sgpr99
+; GFX11-NEXT: ; implicit-def: $sgpr98
+; GFX11-NEXT: ; implicit-def: $sgpr96
+; GFX11-NEXT: ; implicit-def: $sgpr97
+; GFX11-NEXT: ; implicit-def: $sgpr87
+; GFX11-NEXT: ; implicit-def: $sgpr86
+; GFX11-NEXT: ; implicit-def: $sgpr85
+; GFX11-NEXT: ; implicit-def: $sgpr83
+; GFX11-NEXT: ; implicit-def: $sgpr84
+; GFX11-NEXT: ; implicit-def: $sgpr82
+; GFX11-NEXT: ; implicit-def: $sgpr81
+; GFX11-NEXT: ; implicit-def: $sgpr80
+; GFX11-NEXT: ; implicit-def: $sgpr70
+; GFX11-NEXT: ; implicit-def: $sgpr71
+; GFX11-NEXT: ; implicit-def: $sgpr69
+; GFX11-NEXT: ; implicit-def: $sgpr68
+; GFX11-NEXT: ; implicit-def: $sgpr67
+; GFX11-NEXT: ; implicit-def: $sgpr65
+; GFX11-NEXT: ; implicit-def: $sgpr66
+; GFX11-NEXT: ; implicit-def: $sgpr64
+; GFX11-NEXT: ; implicit-def: $sgpr55
+; GFX11-NEXT: ; implicit-def: $sgpr54
+; GFX11-NEXT: ; implicit-def: $sgpr52
+; GFX11-NEXT: ; implicit-def: $sgpr53
+; GFX11-NEXT: ; implicit-def: $sgpr51
+; GFX11-NEXT: ; implicit-def: $sgpr50
+; GFX11-NEXT: ; implicit-def: $sgpr49
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: .LBB95_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s43, s100, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v28, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s26 op_sel_hi:[0,1]
@@ -185880,146 +186902,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v16
; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v15
; GFX11-NEXT: v_lshrrev_b32_e32 v134, 8, v15
-; GFX11-NEXT: s_branch .LBB95_5
-; GFX11-NEXT: .LBB95_3:
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr104
-; GFX11-NEXT: ; implicit-def: $sgpr102
-; GFX11-NEXT: ; implicit-def: $sgpr103
-; GFX11-NEXT: ; implicit-def: $sgpr101
-; GFX11-NEXT: ; implicit-def: $sgpr100
-; GFX11-NEXT: ; implicit-def: $sgpr99
-; GFX11-NEXT: ; implicit-def: $sgpr97
-; GFX11-NEXT: ; implicit-def: $sgpr98
-; GFX11-NEXT: ; implicit-def: $sgpr96
-; GFX11-NEXT: ; implicit-def: $sgpr87
-; GFX11-NEXT: ; implicit-def: $sgpr86
-; GFX11-NEXT: ; implicit-def: $sgpr84
-; GFX11-NEXT: ; implicit-def: $sgpr85
-; GFX11-NEXT: ; implicit-def: $sgpr83
-; GFX11-NEXT: ; implicit-def: $sgpr82
-; GFX11-NEXT: ; implicit-def: $sgpr81
-; GFX11-NEXT: ; implicit-def: $sgpr71
-; GFX11-NEXT: ; implicit-def: $sgpr80
-; GFX11-NEXT: ; implicit-def: $sgpr70
-; GFX11-NEXT: ; implicit-def: $sgpr69
-; GFX11-NEXT: ; implicit-def: $sgpr68
-; GFX11-NEXT: ; implicit-def: $sgpr66
-; GFX11-NEXT: ; implicit-def: $sgpr67
-; GFX11-NEXT: ; implicit-def: $sgpr65
-; GFX11-NEXT: ; implicit-def: $sgpr64
-; GFX11-NEXT: ; implicit-def: $sgpr55
-; GFX11-NEXT: ; implicit-def: $sgpr53
-; GFX11-NEXT: ; implicit-def: $sgpr54
-; GFX11-NEXT: ; implicit-def: $sgpr52
-; GFX11-NEXT: ; implicit-def: $sgpr51
-; GFX11-NEXT: ; implicit-def: $sgpr50
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: s_branch .LBB95_2
-; GFX11-NEXT: .LBB95_4:
+; GFX11-NEXT: s_branch .LBB95_6
+; GFX11-NEXT: .LBB95_5:
; GFX11-NEXT: v_dual_mov_b32 v35, s0 :: v_dual_mov_b32 v36, s1
; GFX11-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-NEXT: v_dual_mov_b32 v15, s28 :: v_dual_mov_b32 v16, s29
@@ -186059,47 +186943,47 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_mov_b32_e32 v57, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 8
; GFX11-NEXT: v_dual_mov_b32 v132, s103 :: v_dual_mov_b32 v135, s101
-; GFX11-NEXT: v_dual_mov_b32 v144, s100 :: v_dual_mov_b32 v131, s99
+; GFX11-NEXT: v_dual_mov_b32 v144, s99 :: v_dual_mov_b32 v131, s98
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v58, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 9
-; GFX11-NEXT: v_dual_mov_b32 v133, s97 :: v_dual_mov_b32 v128, s98
-; GFX11-NEXT: v_dual_mov_b32 v129, s96 :: v_dual_mov_b32 v130, s87
+; GFX11-NEXT: v_dual_mov_b32 v133, s96 :: v_dual_mov_b32 v128, s97
+; GFX11-NEXT: v_dual_mov_b32 v129, s87 :: v_dual_mov_b32 v130, s86
; GFX11-NEXT: v_mov_b32_e32 v59, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 10
-; GFX11-NEXT: v_dual_mov_b32 v116, s86 :: v_dual_mov_b32 v119, s84
-; GFX11-NEXT: v_dual_mov_b32 v114, s85 :: v_dual_mov_b32 v117, s83
+; GFX11-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v119, s83
+; GFX11-NEXT: v_dual_mov_b32 v114, s84 :: v_dual_mov_b32 v117, s82
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v45, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 11
-; GFX11-NEXT: v_dual_mov_b32 v118, s82 :: v_dual_mov_b32 v113, s81
-; GFX11-NEXT: v_dual_mov_b32 v115, s71 :: v_dual_mov_b32 v102, s80
+; GFX11-NEXT: v_dual_mov_b32 v118, s81 :: v_dual_mov_b32 v113, s80
+; GFX11-NEXT: v_dual_mov_b32 v115, s70 :: v_dual_mov_b32 v102, s71
; GFX11-NEXT: v_mov_b32_e32 v47, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 12
-; GFX11-NEXT: v_dual_mov_b32 v103, s70 :: v_dual_mov_b32 v112, s69
-; GFX11-NEXT: v_dual_mov_b32 v98, s68 :: v_dual_mov_b32 v101, s66
+; GFX11-NEXT: v_dual_mov_b32 v103, s69 :: v_dual_mov_b32 v112, s68
+; GFX11-NEXT: v_dual_mov_b32 v98, s67 :: v_dual_mov_b32 v101, s65
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v40, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 13
-; GFX11-NEXT: v_dual_mov_b32 v96, s67 :: v_dual_mov_b32 v99, s65
-; GFX11-NEXT: v_dual_mov_b32 v100, s64 :: v_dual_mov_b32 v87, s55
+; GFX11-NEXT: v_dual_mov_b32 v96, s66 :: v_dual_mov_b32 v99, s64
+; GFX11-NEXT: v_dual_mov_b32 v100, s55 :: v_dual_mov_b32 v87, s54
; GFX11-NEXT: v_mov_b32_e32 v44, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 14
-; GFX11-NEXT: v_dual_mov_b32 v97, s53 :: v_dual_mov_b32 v86, s51
-; GFX11-NEXT: v_dual_mov_b32 v83, s54 :: v_dual_mov_b32 v82, s50
+; GFX11-NEXT: v_dual_mov_b32 v97, s52 :: v_dual_mov_b32 v86, s50
+; GFX11-NEXT: v_dual_mov_b32 v83, s53 :: v_dual_mov_b32 v82, s49
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v46, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 15
-; GFX11-NEXT: v_dual_mov_b32 v85, s52 :: v_dual_mov_b32 v84, s48
-; GFX11-NEXT: v_dual_mov_b32 v55, s49 :: v_dual_mov_b32 v80, s39
+; GFX11-NEXT: v_dual_mov_b32 v85, s51 :: v_dual_mov_b32 v84, s39
+; GFX11-NEXT: v_dual_mov_b32 v55, s48 :: v_dual_mov_b32 v80, s38
; GFX11-NEXT: v_mov_b32_e32 v182, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 16
-; GFX11-NEXT: v_dual_mov_b32 v81, s38 :: v_dual_mov_b32 v32, s37
-; GFX11-NEXT: v_dual_mov_b32 v71, s35 :: v_dual_mov_b32 v26, s36
+; GFX11-NEXT: v_dual_mov_b32 v81, s37 :: v_dual_mov_b32 v32, s36
+; GFX11-NEXT: v_dual_mov_b32 v71, s34 :: v_dual_mov_b32 v26, s35
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v43, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 17
-; GFX11-NEXT: v_dual_mov_b32 v39, s34 :: v_dual_mov_b32 v50, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v39, s31 :: v_dual_mov_b32 v50, s30
; GFX11-NEXT: v_dual_mov_b32 v51, s72 :: v_dual_mov_b32 v52, s62
; GFX11-NEXT: v_mov_b32_e32 v183, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 18
@@ -186118,7 +187002,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_mov_b32_e32 v177, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 21
; GFX11-NEXT: v_mov_b32_e32 v49, s92
-; GFX11-NEXT: v_mov_b32_e32 v67, s30
+; GFX11-NEXT: v_mov_b32_e32 v67, vcc_lo
; GFX11-NEXT: v_mov_b32_e32 v181, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 22
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -186165,7 +187049,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_mov_b32_e32 v147, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v148, s0
-; GFX11-NEXT: .LBB95_5: ; %end
+; GFX11-NEXT: .LBB95_6: ; %end
; GFX11-NEXT: v_perm_b32 v68, v73, v70, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v70, v62, v72, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v69, v60, v69, 0xc0c0004
@@ -192783,361 +193667,363 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s4, v24
; SI-NEXT: v_writelane_b32 v47, s4, 20
; SI-NEXT: v_readfirstlane_b32 s4, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:332
; SI-NEXT: v_writelane_b32 v47, s4, 21
; SI-NEXT: v_readfirstlane_b32 s4, v22
; SI-NEXT: v_writelane_b32 v47, s4, 22
; SI-NEXT: v_readfirstlane_b32 s4, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:328
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:332
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:324
+; SI-NEXT: v_readfirstlane_b32 s92, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:328
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:320
; SI-NEXT: v_writelane_b32 v47, s4, 23
; SI-NEXT: v_readfirstlane_b32 s4, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:320
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:316
; SI-NEXT: v_writelane_b32 v47, s4, 24
-; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:316
-; SI-NEXT: v_writelane_b32 v47, s4, 25
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v47, s4, 26
; SI-NEXT: v_readfirstlane_b32 s4, v17
-; SI-NEXT: v_writelane_b32 v47, s4, 27
+; SI-NEXT: v_writelane_b32 v47, s4, 25
; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v47, s4, 28
+; SI-NEXT: v_writelane_b32 v47, s4, 26
; SI-NEXT: v_readfirstlane_b32 s4, v15
-; SI-NEXT: v_writelane_b32 v47, s4, 29
+; SI-NEXT: v_writelane_b32 v47, s4, 27
; SI-NEXT: v_readfirstlane_b32 s4, v14
-; SI-NEXT: v_writelane_b32 v47, s4, 30
+; SI-NEXT: v_writelane_b32 v47, s4, 28
; SI-NEXT: v_readfirstlane_b32 s4, v13
-; SI-NEXT: v_writelane_b32 v47, s4, 31
-; SI-NEXT: v_readfirstlane_b32 s4, v12
-; SI-NEXT: v_writelane_b32 v47, s4, 32
-; SI-NEXT: v_readfirstlane_b32 s4, v11
-; SI-NEXT: v_writelane_b32 v47, s4, 33
-; SI-NEXT: v_readfirstlane_b32 s4, v10
-; SI-NEXT: v_writelane_b32 v47, s4, 34
+; SI-NEXT: v_writelane_b32 v47, s4, 29
; SI-NEXT: v_readfirstlane_b32 s4, v9
-; SI-NEXT: v_writelane_b32 v47, s4, 35
+; SI-NEXT: v_writelane_b32 v47, s4, 30
; SI-NEXT: v_readfirstlane_b32 s4, v8
-; SI-NEXT: v_writelane_b32 v47, s4, 36
+; SI-NEXT: v_writelane_b32 v47, s4, 31
; SI-NEXT: v_readfirstlane_b32 s4, v7
-; SI-NEXT: v_writelane_b32 v47, s4, 37
+; SI-NEXT: v_writelane_b32 v47, s4, 32
; SI-NEXT: v_readfirstlane_b32 s4, v6
-; SI-NEXT: v_writelane_b32 v47, s4, 38
-; SI-NEXT: v_readfirstlane_b32 s4, v5
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:308
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:304
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:300
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292
+; SI-NEXT: v_readfirstlane_b32 s7, v18
+; SI-NEXT: v_writelane_b32 v47, s4, 33
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
+; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:280
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264
+; SI-NEXT: v_readfirstlane_b32 s11, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:276
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264
-; SI-NEXT: v_writelane_b32 v47, s4, 39
-; SI-NEXT: v_readfirstlane_b32 s4, v4
-; SI-NEXT: v_writelane_b32 v47, s4, 40
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: v_writelane_b32 v47, s4, 41
+; SI-NEXT: v_readfirstlane_b32 s12, v4
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
+; SI-NEXT: v_readfirstlane_b32 s9, v3
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:268
-; SI-NEXT: v_readfirstlane_b32 s69, v2
+; SI-NEXT: v_readfirstlane_b32 s14, v2
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:260
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:308
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:304
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292
+; SI-NEXT: v_readfirstlane_b32 s13, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:300
+; SI-NEXT: v_readfirstlane_b32 s8, v10
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:296
; SI-NEXT: ; implicit-def: $vgpr46 : SGPR spill to VGPR lane
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v23
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v21
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256
+; SI-NEXT: v_writelane_b32 v47, s4, 34
+; SI-NEXT: v_readfirstlane_b32 s4, v19
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:232
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248
-; SI-NEXT: v_writelane_b32 v47, s4, 42
-; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:244
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240
-; SI-NEXT: v_readfirstlane_b32 s83, v21
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:236
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:232
-; SI-NEXT: v_readfirstlane_b32 s86, v22
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:224
-; SI-NEXT: v_readfirstlane_b32 s99, v20
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:248
+; SI-NEXT: v_writelane_b32 v47, s4, 35
+; SI-NEXT: v_readfirstlane_b32 s4, v22
+; SI-NEXT: v_writelane_b32 v47, s4, 36
+; SI-NEXT: v_readfirstlane_b32 s4, v24
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:216
-; SI-NEXT: v_readfirstlane_b32 s17, v19
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216
+; SI-NEXT: v_readfirstlane_b32 s19, v20
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:208
-; SI-NEXT: v_writelane_b32 v47, s4, 43
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:208
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:228
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:224
+; SI-NEXT: v_writelane_b32 v47, s4, 37
+; SI-NEXT: v_readfirstlane_b32 s84, v18
+; SI-NEXT: v_readfirstlane_b32 s95, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s66, v14
-; SI-NEXT: v_readfirstlane_b32 s84, v13
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:196
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
-; SI-NEXT: v_readfirstlane_b32 s8, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:188
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
-; SI-NEXT: v_readfirstlane_b32 s9, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
-; SI-NEXT: v_readfirstlane_b32 s16, v10
-; SI-NEXT: v_readfirstlane_b32 s46, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
-; SI-NEXT: v_readfirstlane_b32 s24, v8
+; SI-NEXT: v_readfirstlane_b32 s68, v8
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:156
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
-; SI-NEXT: v_readfirstlane_b32 s29, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:204
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
-; SI-NEXT: v_readfirstlane_b32 s10, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
-; SI-NEXT: v_readfirstlane_b32 s13, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
-; SI-NEXT: v_readfirstlane_b32 s19, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:164
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
-; SI-NEXT: v_readfirstlane_b32 s47, v5
+; SI-NEXT: v_readfirstlane_b32 s44, v6
+; SI-NEXT: v_readfirstlane_b32 s26, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:136
+; SI-NEXT: v_readfirstlane_b32 s45, v5
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:128
-; SI-NEXT: v_readfirstlane_b32 s27, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:108
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
-; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s74, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:120
+; SI-NEXT: v_readfirstlane_b32 s93, v3
; SI-NEXT: v_readfirstlane_b32 s28, v2
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:96
-; SI-NEXT: v_readfirstlane_b32 s54, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
-; SI-NEXT: v_readfirstlane_b32 s59, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
-; SI-NEXT: v_readfirstlane_b32 s61, v18
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:112
+; SI-NEXT: v_readfirstlane_b32 s29, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:148
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:144
+; SI-NEXT: v_readfirstlane_b32 s64, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:200
+; SI-NEXT: v_readfirstlane_b32 s22, v16
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: v_writelane_b32 v47, s4, 38
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:192
+; SI-NEXT: v_readfirstlane_b32 s24, v9
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:160
+; SI-NEXT: v_readfirstlane_b32 s75, v13
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
+; SI-NEXT: v_readfirstlane_b32 s99, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:176
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:168
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:64
-; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: v_readfirstlane_b32 s52, v21
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:44
+; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_readfirstlane_b32 s66, v21
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52
+; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
+; SI-NEXT: v_readfirstlane_b32 s81, v19
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:40
-; SI-NEXT: v_readfirstlane_b32 s53, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:36
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
-; SI-NEXT: v_readfirstlane_b32 s45, v22
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:28
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
-; SI-NEXT: v_readfirstlane_b32 s60, v20
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:88
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
+; SI-NEXT: v_readfirstlane_b32 s57, v14
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:76
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; SI-NEXT: v_readfirstlane_b32 s83, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:8
-; SI-NEXT: v_readfirstlane_b32 s62, v29
+; SI-NEXT: v_readfirstlane_b32 s86, v29
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84
-; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:80
-; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:56
-; SI-NEXT: v_readfirstlane_b32 s15, v23
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:48
-; SI-NEXT: v_readfirstlane_b32 s56, v25
+; SI-NEXT: v_readfirstlane_b32 s80, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:32
+; SI-NEXT: v_readfirstlane_b32 s65, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:28
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:24
+; SI-NEXT: v_readfirstlane_b32 s46, v25
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:16
-; SI-NEXT: v_writelane_b32 v46, s54, 0
-; SI-NEXT: v_writelane_b32 v46, s53, 1
-; SI-NEXT: v_readfirstlane_b32 s40, v19
-; SI-NEXT: v_readfirstlane_b32 s50, v13
-; SI-NEXT: v_readfirstlane_b32 s64, v27
-; SI-NEXT: v_readfirstlane_b32 s85, v12
-; SI-NEXT: v_readfirstlane_b32 s58, v28
-; SI-NEXT: v_readfirstlane_b32 s57, v11
-; SI-NEXT: v_readfirstlane_b32 s11, v30
-; SI-NEXT: v_readfirstlane_b32 s7, v6
-; SI-NEXT: v_writelane_b32 v46, s7, 2
-; SI-NEXT: v_writelane_b32 v46, s64, 3
-; SI-NEXT: v_writelane_b32 v46, s44, 4
-; SI-NEXT: v_writelane_b32 v46, s85, 5
-; SI-NEXT: v_writelane_b32 v46, s8, 6
-; SI-NEXT: v_writelane_b32 v46, s45, 7
-; SI-NEXT: v_writelane_b32 v46, s46, 8
-; SI-NEXT: v_writelane_b32 v46, s47, 9
-; SI-NEXT: v_writelane_b32 v46, s56, 10
-; SI-NEXT: v_writelane_b32 v46, s9, 11
-; SI-NEXT: v_writelane_b32 v46, s57, 12
-; SI-NEXT: v_writelane_b32 v46, s58, 13
-; SI-NEXT: v_writelane_b32 v46, s10, 14
+; SI-NEXT: v_writelane_b32 v47, s4, 39
+; SI-NEXT: v_readfirstlane_b32 s67, v20
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v0
+; SI-NEXT: v_writelane_b32 v46, vcc_hi, 0
+; SI-NEXT: v_readfirstlane_b32 s34, v53
+; SI-NEXT: v_readfirstlane_b32 s55, v8
+; SI-NEXT: v_readfirstlane_b32 s90, v33
+; SI-NEXT: v_readfirstlane_b32 s27, v11
+; SI-NEXT: v_readfirstlane_b32 s36, v39
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_readfirstlane_b32 s92, v3
-; SI-NEXT: v_writelane_b32 v46, s11, 15
-; SI-NEXT: v_readfirstlane_b32 s12, v10
-; SI-NEXT: v_readfirstlane_b32 s30, v37
-; SI-NEXT: v_writelane_b32 v46, s59, 16
-; SI-NEXT: v_readfirstlane_b32 s36, v7
-; SI-NEXT: v_writelane_b32 v46, s12, 17
-; SI-NEXT: v_readfirstlane_b32 s48, v33
-; SI-NEXT: v_writelane_b32 v46, s60, 18
-; SI-NEXT: v_writelane_b32 v46, s61, 19
-; SI-NEXT: v_writelane_b32 v46, s13, 20
-; SI-NEXT: v_writelane_b32 v46, s62, 21
-; SI-NEXT: v_readfirstlane_b32 s6, v34
-; SI-NEXT: v_writelane_b32 v46, s14, 22
-; SI-NEXT: v_writelane_b32 v46, s15, 23
-; SI-NEXT: v_readfirstlane_b32 s41, v14
-; SI-NEXT: v_readfirstlane_b32 s51, v26
-; SI-NEXT: v_readfirstlane_b32 s87, v31
-; SI-NEXT: v_readfirstlane_b32 s23, v9
-; SI-NEXT: v_readfirstlane_b32 s25, v32
-; SI-NEXT: v_readfirstlane_b32 s26, v8
-; SI-NEXT: v_readfirstlane_b32 s4, v51
-; SI-NEXT: v_writelane_b32 v47, s4, 44
-; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v47, s4, 45
-; SI-NEXT: v_readfirstlane_b32 s4, v52
-; SI-NEXT: v_writelane_b32 v47, s4, 46
-; SI-NEXT: v_readfirstlane_b32 s4, v55
-; SI-NEXT: v_writelane_b32 v47, s4, 47
-; SI-NEXT: v_readfirstlane_b32 s4, v24
-; SI-NEXT: v_writelane_b32 v47, s4, 48
-; SI-NEXT: v_readfirstlane_b32 s4, v40
-; SI-NEXT: v_writelane_b32 v47, s4, 49
+; SI-NEXT: v_readfirstlane_b32 s50, v35
+; SI-NEXT: v_readfirstlane_b32 s49, v5
+; SI-NEXT: v_readfirstlane_b32 s51, v36
+; SI-NEXT: v_readfirstlane_b32 s52, v4
+; SI-NEXT: v_readfirstlane_b32 s54, v37
+; SI-NEXT: v_readfirstlane_b32 s73, v6
+; SI-NEXT: v_readfirstlane_b32 s35, v2
+; SI-NEXT: v_readfirstlane_b32 s37, v48
+; SI-NEXT: v_readfirstlane_b32 s53, v3
+; SI-NEXT: v_readfirstlane_b32 s69, v7
+; SI-NEXT: v_readfirstlane_b32 s72, v34
+; SI-NEXT: v_readfirstlane_b32 s21, v15
+; SI-NEXT: v_readfirstlane_b32 s87, v26
+; SI-NEXT: v_readfirstlane_b32 s70, v38
+; SI-NEXT: v_readfirstlane_b32 s71, v16
+; SI-NEXT: v_readfirstlane_b32 s82, v27
+; SI-NEXT: v_readfirstlane_b32 s4, v32
+; SI-NEXT: v_writelane_b32 v47, s4, 40
+; SI-NEXT: v_writelane_b32 v47, s13, 41
+; SI-NEXT: v_readfirstlane_b32 s79, v13
+; SI-NEXT: v_writelane_b32 v47, s11, 42
+; SI-NEXT: v_writelane_b32 v47, s12, 43
+; SI-NEXT: v_writelane_b32 v47, s6, 44
+; SI-NEXT: v_writelane_b32 v47, s7, 45
+; SI-NEXT: v_writelane_b32 v47, s8, 46
+; SI-NEXT: v_writelane_b32 v47, s9, 47
+; SI-NEXT: v_readfirstlane_b32 s47, v12
+; SI-NEXT: v_writelane_b32 v47, s10, 48
+; SI-NEXT: v_readfirstlane_b32 s56, v28
+; SI-NEXT: v_writelane_b32 v47, s14, 49
+; SI-NEXT: v_readfirstlane_b32 s89, v30
+; SI-NEXT: v_readfirstlane_b32 s91, v10
+; SI-NEXT: v_readfirstlane_b32 s77, v18
+; SI-NEXT: v_readfirstlane_b32 s25, v31
+; SI-NEXT: v_readfirstlane_b32 s98, v9
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v52
+; SI-NEXT: v_readfirstlane_b32 s30, v21
+; SI-NEXT: v_writelane_b32 v46, s30, 1
+; SI-NEXT: v_writelane_b32 v46, s34, 2
+; SI-NEXT: v_writelane_b32 v46, s35, 3
+; SI-NEXT: v_readfirstlane_b32 s60, v54
+; SI-NEXT: v_writelane_b32 v46, s36, 4
+; SI-NEXT: v_readfirstlane_b32 s61, v19
+; SI-NEXT: v_writelane_b32 v46, s60, 5
+; SI-NEXT: v_writelane_b32 v46, s61, 6
+; SI-NEXT: v_readfirstlane_b32 s38, v17
+; SI-NEXT: v_writelane_b32 v46, s37, 7
+; SI-NEXT: s_waitcnt vmcnt(13)
+; SI-NEXT: v_readfirstlane_b32 s39, v1
+; SI-NEXT: v_writelane_b32 v46, s38, 8
+; SI-NEXT: v_readfirstlane_b32 s48, v49
+; SI-NEXT: v_writelane_b32 v46, s39, 9
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s4, v41
-; SI-NEXT: v_writelane_b32 v47, s4, 50
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s4, v43
-; SI-NEXT: v_writelane_b32 v47, s4, 51
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s4, v29
-; SI-NEXT: v_writelane_b32 v47, s4, 52
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s4, v44
-; SI-NEXT: v_writelane_b32 v47, s4, 53
-; SI-NEXT: v_writelane_b32 v47, s92, 54
-; SI-NEXT: v_writelane_b32 v47, s30, 55
-; SI-NEXT: v_writelane_b32 v47, s36, 56
-; SI-NEXT: v_writelane_b32 v47, s48, 57
-; SI-NEXT: v_writelane_b32 v47, s66, 58
-; SI-NEXT: v_writelane_b32 v47, s52, 59
-; SI-NEXT: v_writelane_b32 v47, s84, 60
-; SI-NEXT: v_writelane_b32 v47, s6, 61
-; SI-NEXT: v_readfirstlane_b32 s67, v35
-; SI-NEXT: v_readfirstlane_b32 s94, v5
-; SI-NEXT: v_readfirstlane_b32 s65, v36
-; SI-NEXT: v_readfirstlane_b32 s22, v4
-; SI-NEXT: v_readfirstlane_b32 s18, v38
-; SI-NEXT: v_readfirstlane_b32 s20, v15
-; SI-NEXT: v_readfirstlane_b32 s35, v39
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s78, v48
-; SI-NEXT: v_readfirstlane_b32 s98, v16
-; SI-NEXT: v_readfirstlane_b32 s97, v49
+; SI-NEXT: v_readfirstlane_b32 s62, v50
+; SI-NEXT: v_writelane_b32 v46, s48, 10
+; SI-NEXT: v_writelane_b32 v46, s62, 11
+; SI-NEXT: v_writelane_b32 v46, s49, 12
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_readfirstlane_b32 s63, v14
+; SI-NEXT: v_writelane_b32 v46, s50, 13
+; SI-NEXT: v_writelane_b32 v46, s63, 14
+; SI-NEXT: v_writelane_b32 v46, s51, 15
+; SI-NEXT: v_writelane_b32 v46, s52, 16
+; SI-NEXT: v_writelane_b32 v46, s53, 17
+; SI-NEXT: v_writelane_b32 v46, s54, 18
+; SI-NEXT: v_writelane_b32 v46, s55, 19
+; SI-NEXT: v_writelane_b32 v46, s64, 20
+; SI-NEXT: v_writelane_b32 v46, s81, 21
+; SI-NEXT: v_writelane_b32 v46, s72, 22
+; SI-NEXT: v_writelane_b32 v46, s71, 23
+; SI-NEXT: v_writelane_b32 v46, s87, 24
+; SI-NEXT: v_writelane_b32 v46, s80, 25
+; SI-NEXT: v_writelane_b32 v46, s73, 26
+; SI-NEXT: v_writelane_b32 v46, s82, 27
+; SI-NEXT: v_writelane_b32 v46, s79, 28
+; SI-NEXT: v_writelane_b32 v46, s75, 29
+; SI-NEXT: v_writelane_b32 v46, s65, 30
+; SI-NEXT: v_writelane_b32 v46, s44, 31
+; SI-NEXT: v_writelane_b32 v46, s45, 32
+; SI-NEXT: v_writelane_b32 v46, s46, 33
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s55, v1
+; SI-NEXT: v_readfirstlane_b32 s16, v29
+; SI-NEXT: v_writelane_b32 v46, s47, 34
+; SI-NEXT: v_readfirstlane_b32 s17, v43
+; SI-NEXT: v_writelane_b32 v47, s16, 50
+; SI-NEXT: v_writelane_b32 v46, s56, 35
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s81, v50
-; SI-NEXT: v_readfirstlane_b32 s71, v17
+; SI-NEXT: v_readfirstlane_b32 s15, v44
+; SI-NEXT: v_writelane_b32 v47, s17, 51
+; SI-NEXT: v_writelane_b32 v46, s74, 36
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s96, v0
+; SI-NEXT: v_readfirstlane_b32 s18, v23
+; SI-NEXT: v_writelane_b32 v47, s15, 52
+; SI-NEXT: v_writelane_b32 v46, s89, 37
+; SI-NEXT: v_readfirstlane_b32 s23, v55
+; SI-NEXT: v_writelane_b32 v47, s18, 53
+; SI-NEXT: v_writelane_b32 v46, s57, 38
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s79, v53
+; SI-NEXT: v_readfirstlane_b32 s40, v40
+; SI-NEXT: v_writelane_b32 v47, s23, 54
+; SI-NEXT: v_writelane_b32 v46, s91, 39
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s90, v23
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s37, v54
-; SI-NEXT: v_readfirstlane_b32 s82, v21
-; SI-NEXT: v_readfirstlane_b32 s80, v22
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s70, v25
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s76, v42
-; SI-NEXT: v_readfirstlane_b32 s21, v20
-; SI-NEXT: v_writelane_b32 v47, s50, 62
-; SI-NEXT: v_writelane_b32 v46, s40, 24
-; SI-NEXT: v_writelane_b32 v47, s51, 63
-; SI-NEXT: v_writelane_b32 v46, s41, 25
-; SI-NEXT: s_cbranch_scc0 .LBB97_4
+; SI-NEXT: v_readfirstlane_b32 s41, v22
+; SI-NEXT: v_writelane_b32 v47, s40, 55
+; SI-NEXT: v_writelane_b32 v46, s83, 40
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s42, v25
+; SI-NEXT: v_writelane_b32 v47, s41, 56
+; SI-NEXT: v_writelane_b32 v46, s84, 41
+; SI-NEXT: v_readfirstlane_b32 s43, v41
+; SI-NEXT: v_writelane_b32 v47, s42, 57
+; SI-NEXT: v_writelane_b32 v46, s93, 42
+; SI-NEXT: v_writelane_b32 v47, s43, 58
+; SI-NEXT: v_writelane_b32 v46, s86, 43
+; SI-NEXT: v_readfirstlane_b32 s78, v51
+; SI-NEXT: v_writelane_b32 v47, s77, 59
+; SI-NEXT: v_writelane_b32 v46, s95, 44
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s58, v42
+; SI-NEXT: v_writelane_b32 v47, s78, 60
+; SI-NEXT: v_writelane_b32 v46, s66, 45
+; SI-NEXT: v_readfirstlane_b32 s59, v24
+; SI-NEXT: v_writelane_b32 v47, s58, 61
+; SI-NEXT: v_writelane_b32 v46, s67, 46
+; SI-NEXT: v_writelane_b32 v47, s59, 62
+; SI-NEXT: v_writelane_b32 v46, s21, 47
+; SI-NEXT: v_writelane_b32 v47, vcc_lo, 63
+; SI-NEXT: v_writelane_b32 v46, s31, 48
+; SI-NEXT: s_cbranch_scc0 .LBB97_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_readlane_b32 s4, v47, 13
; SI-NEXT: v_readlane_b32 s5, v47, 12
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 26
+; SI-NEXT: s_or_b32 s20, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 5
; SI-NEXT: v_readlane_b32 s5, v47, 4
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readlane_b32 s5, v47, 41
-; SI-NEXT: v_writelane_b32 v46, s4, 28
-; SI-NEXT: s_and_b32 s4, s69, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: v_writelane_b32 v46, s69, 44
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 30
-; SI-NEXT: v_readlane_b32 s4, v47, 34
-; SI-NEXT: v_readlane_b32 s5, v47, 33
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: v_readlane_b32 s4, v47, 26
-; SI-NEXT: v_readlane_b32 s5, v47, 25
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 34
+; SI-NEXT: s_or_b32 s88, s4, s5
+; SI-NEXT: s_and_b32 s4, s14, 0xff
+; SI-NEXT: s_lshl_b32 s5, s9, 8
+; SI-NEXT: s_or_b32 s94, s4, s5
+; SI-NEXT: s_and_b32 s4, s8, 0xff
+; SI-NEXT: s_lshl_b32 s5, s6, 8
+; SI-NEXT: s_or_b32 s85, s4, s5
+; SI-NEXT: s_and_b32 s4, s7, 0xff
+; SI-NEXT: s_lshl_b32 s5, s92, 8
+; SI-NEXT: s_or_b32 s96, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 18
; SI-NEXT: v_readlane_b32 s5, v47, 17
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s5, s76, 8
-; SI-NEXT: v_writelane_b32 v46, s24, 45
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: s_and_b32 s4, s82, 0xff
-; SI-NEXT: s_lshl_b32 s5, s37, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: s_and_b32 s4, s71, 0xff
-; SI-NEXT: s_lshl_b32 s5, s81, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 42
-; SI-NEXT: s_and_b32 s4, s20, 0xff
-; SI-NEXT: s_lshl_b32 s5, s18, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 46
-; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: s_lshl_b32 s5, s6, 8
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_writelane_b32 v46, s4, 47
-; SI-NEXT: s_and_b32 s4, s12, 0xff
-; SI-NEXT: s_lshl_b32 s5, s11, 8
+; SI-NEXT: v_writelane_b32 v46, s22, 49
+; SI-NEXT: s_or_b32 s22, s4, s5
+; SI-NEXT: s_and_b32 s4, s59, 0xff
+; SI-NEXT: s_lshl_b32 s5, s58, 8
+; SI-NEXT: s_or_b32 s58, s4, s5
+; SI-NEXT: s_and_b32 s4, s61, 0xff
+; SI-NEXT: s_lshl_b32 s5, s60, 8
+; SI-NEXT: s_or_b32 s59, s4, s5
+; SI-NEXT: s_and_b32 s4, s63, 0xff
+; SI-NEXT: s_lshl_b32 s5, s62, 8
+; SI-NEXT: s_or_b32 s60, s4, s5
+; SI-NEXT: s_and_b32 s4, s27, 0xff
+; SI-NEXT: s_lshl_b32 s5, s70, 8
+; SI-NEXT: s_or_b32 s61, s4, s5
+; SI-NEXT: s_and_b32 s4, s73, 0xff
+; SI-NEXT: s_lshl_b32 s5, s72, 8
+; SI-NEXT: s_or_b32 s62, s4, s5
+; SI-NEXT: s_and_b32 s4, s91, 0xff
+; SI-NEXT: s_lshl_b32 s5, s89, 8
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_and_b32 s4, s41, 0xff
-; SI-NEXT: s_lshl_b32 s5, s40, 8
+; SI-NEXT: s_and_b32 s4, s21, 0xff
+; SI-NEXT: s_lshl_b32 s5, s67, 8
; SI-NEXT: s_or_b32 s72, s4, s5
-; SI-NEXT: s_and_b32 s4, s15, 0xff
-; SI-NEXT: s_lshl_b32 s5, s14, 8
+; SI-NEXT: s_and_b32 s4, s66, 0xff
+; SI-NEXT: s_lshl_b32 s5, s95, 8
; SI-NEXT: s_or_b32 s73, s4, s5
-; SI-NEXT: s_and_b32 s4, s13, 0xff
-; SI-NEXT: s_lshl_b32 s5, s10, 8
+; SI-NEXT: s_and_b32 s4, s93, 0xff
+; SI-NEXT: s_lshl_b32 s5, s74, 8
; SI-NEXT: s_or_b32 s74, s4, s5
-; SI-NEXT: s_and_b32 s4, s9, 0xff
-; SI-NEXT: s_lshl_b32 s5, s8, 8
+; SI-NEXT: s_and_b32 s4, s99, 0xff
+; SI-NEXT: s_lshl_b32 s5, s75, 8
; SI-NEXT: s_or_b32 s75, s4, s5
; SI-NEXT: v_readlane_b32 s4, v47, 9
; SI-NEXT: v_readlane_b32 s5, v47, 8
@@ -193149,61 +194035,63 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_readlane_b32 s6, v47, 6
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s6, s6, 24
-; SI-NEXT: s_or_b32 s68, s6, s4
+; SI-NEXT: s_or_b32 s7, s6, s4
; SI-NEXT: v_readlane_b32 s4, v47, 11
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: v_readlane_b32 s6, v47, 10
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: v_writelane_b32 v46, s7, 50
; SI-NEXT: s_or_b32 s4, s6, s4
+; SI-NEXT: s_or_b32 s5, s5, s7
; SI-NEXT: v_readlane_b32 s6, v47, 1
; SI-NEXT: v_readlane_b32 s7, v47, 0
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
; SI-NEXT: s_or_b32 s7, s6, s7
-; SI-NEXT: v_readlane_b32 s6, v47, 43
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v47, 42
+; SI-NEXT: s_and_b32 s6, s10, 0xff
+; SI-NEXT: v_readlane_b32 s8, v47, 34
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_or_b32 s91, s8, s6
+; SI-NEXT: s_or_b32 s9, s8, s6
; SI-NEXT: v_readlane_b32 s6, v47, 3
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: v_readlane_b32 s8, v47, 2
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: v_writelane_b32 v46, s9, 51
; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_readlane_b32 s8, v47, 38
-; SI-NEXT: v_readlane_b32 s9, v47, 37
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: v_readlane_b32 s8, v47, 33
+; SI-NEXT: v_readlane_b32 s9, v47, 32
; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 8
; SI-NEXT: s_or_b32 s9, s8, s9
-; SI-NEXT: v_readlane_b32 s8, v47, 36
+; SI-NEXT: v_readlane_b32 s8, v47, 31
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v47, 35
+; SI-NEXT: v_readlane_b32 s10, v47, 30
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_lshl_b32 s10, s10, 24
-; SI-NEXT: s_or_b32 s88, s10, s8
-; SI-NEXT: v_readlane_b32 s8, v47, 40
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v47, 39
+; SI-NEXT: s_or_b32 s14, s10, s8
+; SI-NEXT: s_and_b32 s8, s12, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_lshl_b32 s10, s11, 24
; SI-NEXT: s_or_b32 s8, s10, s8
-; SI-NEXT: v_readlane_b32 s10, v47, 30
-; SI-NEXT: v_readlane_b32 s11, v47, 29
+; SI-NEXT: v_readlane_b32 s10, v47, 28
+; SI-NEXT: v_readlane_b32 s11, v47, 27
; SI-NEXT: s_and_b32 s10, s10, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 8
; SI-NEXT: s_or_b32 s11, s10, s11
-; SI-NEXT: v_readlane_b32 s10, v47, 28
+; SI-NEXT: v_readlane_b32 s10, v47, 26
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v47, 27
+; SI-NEXT: v_readlane_b32 s12, v47, 25
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s12, s12, 24
-; SI-NEXT: s_or_b32 s95, s12, s10
-; SI-NEXT: v_readlane_b32 s10, v47, 32
-; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v47, 31
+; SI-NEXT: s_or_b32 s91, s12, s10
+; SI-NEXT: s_and_b32 s10, s13, 0xff
+; SI-NEXT: v_readlane_b32 s12, v47, 29
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_or_b32 s10, s12, s10
@@ -193211,13 +194099,16 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_readlane_b32 s13, v47, 21
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 8
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s13, s12, s13
; SI-NEXT: v_readlane_b32 s12, v47, 20
+; SI-NEXT: v_writelane_b32 v46, s14, 52
+; SI-NEXT: s_or_b32 s9, s9, s14
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: v_readlane_b32 s14, v47, 19
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_or_b32 s89, s14, s12
+; SI-NEXT: s_or_b32 s93, s14, s12
; SI-NEXT: v_readlane_b32 s12, v47, 24
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: v_readlane_b32 s14, v47, 23
@@ -193225,366 +194116,337 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s14, s14, 24
; SI-NEXT: s_or_b32 s12, s14, s12
; SI-NEXT: v_readlane_b32 s14, v47, 14
-; SI-NEXT: v_readlane_b32 s15, v47, 53
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: v_writelane_b32 v46, s71, 48
; SI-NEXT: s_or_b32 s15, s14, s15
-; SI-NEXT: v_readlane_b32 s14, v47, 52
-; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_writelane_b32 v46, s16, 49
-; SI-NEXT: v_readlane_b32 s16, v47, 51
+; SI-NEXT: s_and_b32 s14, s16, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: v_writelane_b32 v46, s18, 50
-; SI-NEXT: s_lshl_b32 s18, s16, 24
-; SI-NEXT: s_or_b32 s93, s18, s14
+; SI-NEXT: s_mov_b32 s67, s29
+; SI-NEXT: s_mov_b32 s29, s28
+; SI-NEXT: s_mov_b32 s28, s26
+; SI-NEXT: s_mov_b32 s26, s68
+; SI-NEXT: s_mov_b32 s68, s24
+; SI-NEXT: s_mov_b32 s24, s25
+; SI-NEXT: s_lshl_b32 s25, s17, 24
+; SI-NEXT: s_or_b32 s17, s25, s14
; SI-NEXT: v_readlane_b32 s14, v47, 16
+; SI-NEXT: v_writelane_b32 v46, s90, 53
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: v_readlane_b32 s16, v47, 15
+; SI-NEXT: v_writelane_b32 v46, s69, 54
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s18, s16, 24
-; SI-NEXT: v_readlane_b32 s16, v47, 49
-; SI-NEXT: s_or_b32 s14, s18, s14
-; SI-NEXT: s_and_b32 s18, s80, 0xff
-; SI-NEXT: s_lshl_b32 s40, s16, 8
-; SI-NEXT: v_readlane_b32 s16, v47, 48
-; SI-NEXT: s_or_b32 s41, s18, s40
-; SI-NEXT: s_and_b32 s18, s16, 0xff
-; SI-NEXT: v_readlane_b32 s16, v47, 47
-; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_lshl_b32 s40, s16, 24
-; SI-NEXT: s_mov_b32 s24, s21
-; SI-NEXT: s_or_b32 s21, s40, s18
-; SI-NEXT: s_and_b32 s40, s70, 0xff
-; SI-NEXT: v_readlane_b32 s18, v47, 50
-; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: s_lshl_b32 s42, s18, 24
-; SI-NEXT: v_readlane_b32 s18, v47, 46
-; SI-NEXT: s_or_b32 s40, s42, s40
-; SI-NEXT: s_and_b32 s42, s96, 0xff
-; SI-NEXT: s_lshl_b32 s43, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v47, 45
-; SI-NEXT: s_or_b32 s43, s42, s43
-; SI-NEXT: s_and_b32 s42, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v47, 44
-; SI-NEXT: s_mov_b32 s34, s76
-; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s76, s18, 24
-; SI-NEXT: s_or_b32 s69, s76, s42
-; SI-NEXT: s_and_b32 s42, s90, 0xff
+; SI-NEXT: s_lshl_b32 s25, s16, 24
+; SI-NEXT: s_or_b32 s14, s25, s14
+; SI-NEXT: s_and_b32 s25, s41, 0xff
+; SI-NEXT: v_writelane_b32 v46, s27, 55
+; SI-NEXT: s_lshl_b32 s27, s40, 8
+; SI-NEXT: s_or_b32 s25, s25, s27
+; SI-NEXT: s_and_b32 s27, s18, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s18, s23, 24
+; SI-NEXT: s_or_b32 s18, s18, s27
+; SI-NEXT: s_and_b32 s27, s42, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s40, s43, 24
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_or_b32 s40, s40, s27
+; SI-NEXT: s_or_b32 s41, s25, s18
+; SI-NEXT: s_and_b32 s25, vcc_hi, 0xff
+; SI-NEXT: s_lshl_b32 s27, vcc_lo, 8
+; SI-NEXT: s_or_b32 s27, s25, s27
+; SI-NEXT: s_and_b32 s25, s77, 0xff
+; SI-NEXT: s_lshl_b32 s25, s25, 16
+; SI-NEXT: s_lshl_b32 s42, s78, 24
+; SI-NEXT: s_or_b32 s66, s42, s25
+; SI-NEXT: s_and_b32 s42, s30, 0xff
; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_lshl_b32 s76, s79, 24
-; SI-NEXT: s_or_b32 s42, s76, s42
-; SI-NEXT: s_and_b32 s76, s98, 0xff
-; SI-NEXT: s_lshl_b32 s77, s78, 8
-; SI-NEXT: v_writelane_b32 v46, s20, 51
+; SI-NEXT: s_lshl_b32 s43, s34, 24
+; SI-NEXT: s_and_b32 s27, s27, 0xffff
+; SI-NEXT: s_or_b32 s42, s43, s42
+; SI-NEXT: s_or_b32 s43, s27, s66
+; SI-NEXT: s_and_b32 s27, s38, 0xff
+; SI-NEXT: s_lshl_b32 s76, s37, 8
+; SI-NEXT: s_or_b32 s76, s27, s76
+; SI-NEXT: s_and_b32 s27, s35, 0xff
+; SI-NEXT: s_lshl_b32 s27, s27, 16
+; SI-NEXT: s_lshl_b32 s77, s36, 24
+; SI-NEXT: s_or_b32 s90, s77, s27
+; SI-NEXT: s_and_b32 s77, s39, 0xff
+; SI-NEXT: s_lshl_b32 s77, s77, 16
+; SI-NEXT: s_lshl_b32 s78, s48, 24
+; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: s_or_b32 s34, s78, s77
+; SI-NEXT: s_or_b32 s35, s76, s90
+; SI-NEXT: s_and_b32 s76, s52, 0xff
+; SI-NEXT: s_lshl_b32 s77, s51, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s31, 0xff
-; SI-NEXT: v_writelane_b32 v46, s78, 52
+; SI-NEXT: s_and_b32 s77, s49, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s35, 24
-; SI-NEXT: s_mov_b32 s16, s70
-; SI-NEXT: s_or_b32 s70, s78, s77
-; SI-NEXT: s_and_b32 s77, s55, 0xff
+; SI-NEXT: s_lshl_b32 s78, s50, 24
+; SI-NEXT: s_or_b32 s69, s78, s77
+; SI-NEXT: s_and_b32 s77, s53, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s97, 24
+; SI-NEXT: s_lshl_b32 s78, s54, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_or_b32 vcc_lo, s78, s77
-; SI-NEXT: s_or_b32 vcc_hi, s76, s70
-; SI-NEXT: s_and_b32 s76, s22, 0xff
-; SI-NEXT: s_lshl_b32 s77, s65, 8
+; SI-NEXT: s_or_b32 s36, s78, s77
+; SI-NEXT: s_or_b32 s37, s76, s69
+; SI-NEXT: s_and_b32 s76, s55, 0xff
+; SI-NEXT: s_lshl_b32 s77, s98, 8
+; SI-NEXT: v_readlane_b32 s16, v47, 40
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s94, 0xff
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_writelane_b32 v46, s70, 56
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s67, 24
-; SI-NEXT: s_or_b32 s71, s78, s77
-; SI-NEXT: s_and_b32 s77, s92, 0xff
+; SI-NEXT: s_lshl_b32 s78, s24, 24
+; SI-NEXT: v_readlane_b32 s16, v46, 54
+; SI-NEXT: s_or_b32 s70, s78, s77
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v46, 53
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s30, 24
+; SI-NEXT: s_lshl_b32 s78, s16, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s38, s78, s77
-; SI-NEXT: s_or_b32 s39, s76, s71
-; SI-NEXT: s_and_b32 s76, s26, 0xff
-; SI-NEXT: s_lshl_b32 s77, s25, 8
+; SI-NEXT: s_or_b32 s39, s76, s70
+; SI-NEXT: s_and_b32 s76, s79, 0xff
+; SI-NEXT: s_lshl_b32 s77, s82, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s23, 0xff
+; SI-NEXT: s_and_b32 s77, s71, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
; SI-NEXT: s_lshl_b32 s78, s87, 24
-; SI-NEXT: v_writelane_b32 v46, s80, 53
-; SI-NEXT: s_or_b32 s80, s78, s77
-; SI-NEXT: s_and_b32 s77, s36, 0xff
+; SI-NEXT: s_or_b32 s71, s78, s77
+; SI-NEXT: s_and_b32 s77, s47, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s48, 24
+; SI-NEXT: s_lshl_b32 s78, s56, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s48, s78, s77
-; SI-NEXT: s_or_b32 s49, s76, s80
-; SI-NEXT: s_and_b32 s76, s85, 0xff
-; SI-NEXT: s_lshl_b32 s77, s64, 8
-; SI-NEXT: v_writelane_b32 v46, s98, 54
+; SI-NEXT: s_or_b32 s49, s76, s71
+; SI-NEXT: s_and_b32 s76, s46, 0xff
+; SI-NEXT: s_lshl_b32 s77, s65, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s50, 0xff
-; SI-NEXT: v_writelane_b32 v46, s24, 55
+; SI-NEXT: s_and_b32 s77, s80, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s51, 24
-; SI-NEXT: v_writelane_b32 v46, s34, 56
-; SI-NEXT: v_writelane_b32 v46, s81, 57
-; SI-NEXT: s_or_b32 s81, s78, s77
-; SI-NEXT: s_and_b32 s77, s57, 0xff
+; SI-NEXT: s_lshl_b32 s78, s81, 24
+; SI-NEXT: s_or_b32 vcc_hi, s78, s77
+; SI-NEXT: s_and_b32 s77, s86, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s58, 24
+; SI-NEXT: s_lshl_b32 s78, s83, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v47, 39
; SI-NEXT: s_or_b32 s50, s78, s77
-; SI-NEXT: s_or_b32 s51, s76, s81
-; SI-NEXT: s_and_b32 s76, s56, 0xff
-; SI-NEXT: s_lshl_b32 s77, s45, 8
-; SI-NEXT: v_writelane_b32 v46, s22, 58
+; SI-NEXT: s_or_b32 s51, s76, vcc_hi
+; SI-NEXT: s_and_b32 s76, s31, 0xff
+; SI-NEXT: s_lshl_b32 s77, s16, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s53, 0xff
-; SI-NEXT: v_writelane_b32 v46, s65, 59
+; SI-NEXT: s_and_b32 s77, s29, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s52, 24
-; SI-NEXT: v_writelane_b32 v46, s67, 60
-; SI-NEXT: v_writelane_b32 v46, s94, 61
-; SI-NEXT: s_mov_b32 s94, s82
-; SI-NEXT: s_or_b32 s82, s78, s77
-; SI-NEXT: s_and_b32 s77, s62, 0xff
+; SI-NEXT: s_lshl_b32 s78, s28, 24
+; SI-NEXT: s_or_b32 s31, s78, s77
+; SI-NEXT: s_and_b32 s77, s84, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s60, 24
+; SI-NEXT: s_lshl_b32 s78, s57, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
; SI-NEXT: s_or_b32 s52, s78, s77
-; SI-NEXT: s_or_b32 s53, s76, s82
-; SI-NEXT: s_and_b32 s76, s44, 0xff
-; SI-NEXT: s_lshl_b32 s77, s54, 8
+; SI-NEXT: s_or_b32 s53, s76, s31
+; SI-NEXT: s_and_b32 s76, s67, 0xff
+; SI-NEXT: s_lshl_b32 s77, s26, 8
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s28, 0xff
+; SI-NEXT: s_and_b32 s77, s68, 0xff
+; SI-NEXT: v_readlane_b32 s16, v47, 38
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s27, 24
-; SI-NEXT: v_writelane_b32 v46, s97, 62
-; SI-NEXT: s_mov_b32 s92, s28
-; SI-NEXT: s_mov_b32 s28, s27
-; SI-NEXT: s_mov_b32 s27, s26
-; SI-NEXT: s_mov_b32 s26, s25
-; SI-NEXT: s_mov_b32 s25, s87
-; SI-NEXT: s_or_b32 s87, s78, s77
-; SI-NEXT: s_and_b32 s77, s61, 0xff
-; SI-NEXT: s_mov_b32 s24, s23
-; SI-NEXT: s_mov_b32 s23, s16
+; SI-NEXT: s_lshl_b32 s78, s16, 24
+; SI-NEXT: s_or_b32 s82, s78, s77
+; SI-NEXT: s_and_b32 s77, s45, 0xff
; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s59, 24
+; SI-NEXT: s_lshl_b32 s78, s44, 24
; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: v_readlane_b32 s16, v46, 45
-; SI-NEXT: s_mov_b32 s85, s55
+; SI-NEXT: v_readlane_b32 s16, v47, 37
; SI-NEXT: s_or_b32 s54, s78, s77
-; SI-NEXT: s_or_b32 s55, s76, s87
-; SI-NEXT: s_and_b32 s76, s29, 0xff
+; SI-NEXT: s_or_b32 s55, s76, s82
+; SI-NEXT: s_and_b32 s76, s19, 0xff
; SI-NEXT: s_lshl_b32 s77, s16, 8
-; SI-NEXT: v_readlane_b32 s18, v46, 26
+; SI-NEXT: v_readlane_b32 s16, v47, 36
+; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s19, 0xff
-; SI-NEXT: v_readlane_b32 s16, v46, 49
-; SI-NEXT: s_and_b32 s44, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 28
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: v_readlane_b32 s16, v47, 35
+; SI-NEXT: s_or_b32 s15, s15, s17
; SI-NEXT: s_lshl_b32 s77, s77, 16
; SI-NEXT: s_lshl_b32 s78, s16, 24
-; SI-NEXT: s_and_b32 s45, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 30
-; SI-NEXT: s_mov_b32 s20, s19
-; SI-NEXT: s_mov_b32 s19, s37
-; SI-NEXT: s_mov_b32 s37, s35
-; SI-NEXT: s_mov_b32 s35, s31
-; SI-NEXT: s_mov_b32 s31, s79
-; SI-NEXT: s_mov_b32 s79, s90
-; SI-NEXT: s_mov_b32 s90, s96
-; SI-NEXT: s_or_b32 s96, s78, s77
-; SI-NEXT: s_lshl_b32 s78, s46, 24
-; SI-NEXT: s_and_b32 s46, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 32
-; SI-NEXT: s_and_b32 s77, s47, 0xff
-; SI-NEXT: s_and_b32 s47, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 34
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_and_b32 s56, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 36
-; SI-NEXT: s_or_b32 s64, s78, s77
-; SI-NEXT: s_or_b32 s65, s76, s96
-; SI-NEXT: s_and_b32 s76, s17, 0xff
-; SI-NEXT: s_lshl_b32 s77, s99, 8
-; SI-NEXT: s_and_b32 s57, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 38
-; SI-NEXT: s_or_b32 s76, s76, s77
-; SI-NEXT: s_and_b32 s77, s86, 0xff
-; SI-NEXT: s_and_b32 s30, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 40
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s83, 24
-; SI-NEXT: s_and_b32 s34, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 42
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_mov_b32 s16, s83
-; SI-NEXT: s_or_b32 s83, s78, s77
-; SI-NEXT: s_and_b32 s77, s84, 0xff
-; SI-NEXT: s_and_b32 s36, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 46
-; SI-NEXT: s_or_b32 s5, s5, s68
-; SI-NEXT: s_lshl_b32 s77, s77, 16
-; SI-NEXT: s_lshl_b32 s78, s66, 24
-; SI-NEXT: s_and_b32 s76, s76, 0xffff
-; SI-NEXT: s_and_b32 s84, s18, 0xffff
-; SI-NEXT: v_readlane_b32 s18, v46, 47
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s66, s78, s77
-; SI-NEXT: s_or_b32 s67, s76, s83
-; SI-NEXT: s_mov_b32 s78, s86
-; SI-NEXT: s_and_b32 s86, s18, 0xffff
-; SI-NEXT: s_mov_b32 s76, s99
-; SI-NEXT: s_and_b32 s99, s74, 0xffff
-; SI-NEXT: s_and_b32 s18, s75, 0xffff
+; SI-NEXT: v_readlane_b32 s16, v46, 49
+; SI-NEXT: s_and_b32 s44, s20, 0xffff
+; SI-NEXT: s_mov_b32 s25, s99
+; SI-NEXT: s_or_b32 s87, s78, s77
+; SI-NEXT: s_and_b32 s77, s16, 0xff
+; SI-NEXT: s_and_b32 s57, s22, 0xffff
+; SI-NEXT: s_and_b32 s22, s74, 0xffff
+; SI-NEXT: s_and_b32 s99, s75, 0xffff
; SI-NEXT: s_or_b32 s74, s44, s4
; SI-NEXT: s_mov_b32 s75, s5
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_or_b32 s7, s7, s91
-; SI-NEXT: v_writelane_b32 v46, s4, 26
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 27
-; SI-NEXT: s_lshr_b64 s[4:5], s[6:7], 16
-; SI-NEXT: s_or_b32 s9, s9, s88
-; SI-NEXT: v_writelane_b32 v46, s4, 28
+; SI-NEXT: s_lshr_b64 s[20:21], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[14:15], 16
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 29
-; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 16
-; SI-NEXT: s_or_b32 s11, s11, s95
-; SI-NEXT: v_writelane_b32 v46, s4, 30
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 31
-; SI-NEXT: s_lshr_b64 s[4:5], s[10:11], 16
-; SI-NEXT: s_or_b32 s13, s13, s89
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 33
-; SI-NEXT: s_lshr_b64 s[4:5], s[12:13], 16
-; SI-NEXT: s_or_b32 s15, s15, s93
-; SI-NEXT: v_writelane_b32 v46, s4, 34
-; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 35
-; SI-NEXT: s_lshr_b64 s[4:5], s[14:15], 16
-; SI-NEXT: s_or_b32 s41, s41, s21
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s5, 37
-; SI-NEXT: s_lshr_b64 s[4:5], s[40:41], 16
-; SI-NEXT: s_or_b32 s43, s43, s69
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: v_writelane_b32 v46, s5, 39
-; SI-NEXT: s_lshr_b64 s[4:5], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: v_writelane_b32 v46, s5, 41
-; SI-NEXT: s_lshr_b64 s[4:5], vcc, 16
-; SI-NEXT: s_and_b32 s97, s63, 0xffff
-; SI-NEXT: s_and_b32 s98, s72, 0xffff
-; SI-NEXT: s_and_b32 s22, s73, 0xffff
-; SI-NEXT: v_writelane_b32 v46, s4, 42
+; SI-NEXT: s_lshl_b32 s77, s77, 16
+; SI-NEXT: s_lshl_b32 s78, s64, 24
+; SI-NEXT: s_and_b32 s76, s76, 0xffff
+; SI-NEXT: v_readlane_b32 s5, v46, 50
+; SI-NEXT: s_or_b32 s11, s11, s91
+; SI-NEXT: s_or_b32 s13, s13, s93
+; SI-NEXT: s_mov_b32 s27, s98
+; SI-NEXT: s_or_b32 s64, s78, s77
+; SI-NEXT: s_or_b32 s65, s76, s87
+; SI-NEXT: s_and_b32 s45, s88, 0xffff
+; SI-NEXT: s_and_b32 s46, s94, 0xffff
+; SI-NEXT: s_and_b32 s47, s85, 0xffff
+; SI-NEXT: s_and_b32 s56, s96, 0xffff
+; SI-NEXT: s_and_b32 vcc_lo, s58, 0xffff
+; SI-NEXT: s_and_b32 s30, s59, 0xffff
+; SI-NEXT: s_and_b32 s96, s60, 0xffff
+; SI-NEXT: s_and_b32 s83, s61, 0xffff
+; SI-NEXT: s_and_b32 s16, s62, 0xffff
+; SI-NEXT: s_and_b32 s86, s63, 0xffff
+; SI-NEXT: s_and_b32 s97, s72, 0xffff
+; SI-NEXT: s_and_b32 s98, s73, 0xffff
+; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 16
+; SI-NEXT: s_lshr_b32 s23, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v46, 51
; SI-NEXT: s_or_b32 s72, s45, s6
; SI-NEXT: s_mov_b32 s73, s7
; SI-NEXT: s_or_b32 s62, s46, s8
; SI-NEXT: s_mov_b32 s63, s9
+; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
; SI-NEXT: s_or_b32 s60, s47, s10
; SI-NEXT: s_mov_b32 s61, s11
+; SI-NEXT: s_lshr_b64 s[80:81], s[10:11], 16
; SI-NEXT: s_or_b32 s58, s56, s12
; SI-NEXT: s_mov_b32 s59, s13
+; SI-NEXT: s_lshr_b64 s[10:11], s[12:13], 16
; SI-NEXT: s_or_b32 s56, s57, s14
; SI-NEXT: s_mov_b32 s57, s15
-; SI-NEXT: s_or_b32 s46, s30, s40
+; SI-NEXT: s_or_b32 s46, vcc_lo, s40
; SI-NEXT: s_mov_b32 s47, s41
-; SI-NEXT: v_writelane_b32 v46, s5, 43
-; SI-NEXT: s_or_b32 s40, s84, s38
-; SI-NEXT: s_mov_b32 s41, s39
+; SI-NEXT: s_lshr_b64 s[6:7], s[40:41], 16
+; SI-NEXT: s_or_b32 s44, s30, s42
+; SI-NEXT: s_mov_b32 s45, s43
+; SI-NEXT: s_lshr_b64 s[88:89], s[42:43], 16
+; SI-NEXT: s_or_b32 s42, s96, s34
+; SI-NEXT: s_mov_b32 s43, s35
+; SI-NEXT: s_lshr_b64 s[84:85], s[34:35], 16
+; SI-NEXT: s_or_b32 s40, s83, s36
+; SI-NEXT: s_mov_b32 s41, s37
+; SI-NEXT: s_lshr_b64 s[8:9], s[36:37], 16
+; SI-NEXT: s_or_b32 s14, s16, s38
+; SI-NEXT: s_mov_b32 s15, s39
; SI-NEXT: s_lshr_b64 s[38:39], s[38:39], 16
-; SI-NEXT: s_or_b32 s14, s86, s48
-; SI-NEXT: s_mov_b32 s15, s49
+; SI-NEXT: s_or_b32 s12, s86, s48
+; SI-NEXT: s_mov_b32 s13, s49
; SI-NEXT: s_lshr_b64 s[48:49], s[48:49], 16
-; SI-NEXT: s_or_b32 s12, s97, s50
-; SI-NEXT: s_mov_b32 s13, s51
+; SI-NEXT: s_or_b32 s94, s97, s50
+; SI-NEXT: s_mov_b32 s95, s51
; SI-NEXT: s_lshr_b64 s[50:51], s[50:51], 16
-; SI-NEXT: s_or_b32 s10, s98, s52
-; SI-NEXT: s_mov_b32 s11, s53
+; SI-NEXT: s_or_b32 s36, s98, s52
+; SI-NEXT: s_mov_b32 s37, s53
; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
-; SI-NEXT: s_or_b32 s8, s22, s54
-; SI-NEXT: s_mov_b32 s9, s55
+; SI-NEXT: s_or_b32 s34, s22, s54
+; SI-NEXT: s_mov_b32 s35, s55
; SI-NEXT: s_lshr_b64 s[54:55], s[54:55], 16
-; SI-NEXT: s_or_b32 s6, s99, s64
-; SI-NEXT: s_mov_b32 s7, s65
+; SI-NEXT: s_lshr_b32 s77, s5, 16
+; SI-NEXT: v_readlane_b32 s5, v46, 52
+; SI-NEXT: v_readlane_b32 s22, v46, 49
+; SI-NEXT: s_or_b32 s96, s99, s64
+; SI-NEXT: s_mov_b32 s97, s65
; SI-NEXT: s_lshr_b64 s[64:65], s[64:65], 16
-; SI-NEXT: s_or_b32 s4, s18, s66
-; SI-NEXT: s_mov_b32 s5, s67
-; SI-NEXT: s_lshr_b64 s[66:67], s[66:67], 16
-; SI-NEXT: s_or_b32 s44, s34, s42
-; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_or_b32 s42, s36, vcc_lo
-; SI-NEXT: s_mov_b32 s43, vcc_hi
-; SI-NEXT: s_mov_b32 s86, s78
-; SI-NEXT: s_mov_b32 s99, s76
-; SI-NEXT: v_readlane_b32 s18, v46, 50
-; SI-NEXT: s_lshr_b32 s77, s68, 16
-; SI-NEXT: s_lshr_b32 s68, s91, 16
-; SI-NEXT: s_lshr_b32 s88, s88, 16
-; SI-NEXT: s_lshr_b32 s84, s95, 16
-; SI-NEXT: s_lshr_b32 s89, s89, 16
-; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s93, s21, 16
-; SI-NEXT: s_lshr_b32 s95, s69, 16
-; SI-NEXT: s_lshr_b32 s30, s70, 16
-; SI-NEXT: s_lshr_b32 s34, s71, 16
-; SI-NEXT: s_lshr_b32 s36, s80, 16
-; SI-NEXT: v_readlane_b32 s69, v46, 44
-; SI-NEXT: v_readlane_b32 s80, v46, 53
-; SI-NEXT: s_lshr_b32 s39, s81, 16
-; SI-NEXT: v_readlane_b32 s81, v46, 57
-; SI-NEXT: v_readlane_b32 s76, v46, 56
-; SI-NEXT: v_readlane_b32 s21, v46, 55
-; SI-NEXT: v_readlane_b32 s98, v46, 54
-; SI-NEXT: s_lshr_b32 s49, s82, 16
-; SI-NEXT: s_mov_b32 s82, s94
-; SI-NEXT: v_readlane_b32 s71, v46, 48
-; SI-NEXT: v_readlane_b32 s78, v46, 52
-; SI-NEXT: s_mov_b32 s55, s85
-; SI-NEXT: v_readlane_b32 s97, v46, 62
-; SI-NEXT: v_readlane_b32 s94, v46, 61
-; SI-NEXT: v_readlane_b32 s67, v46, 60
-; SI-NEXT: v_readlane_b32 s65, v46, 59
-; SI-NEXT: v_readlane_b32 s22, v46, 58
-; SI-NEXT: s_lshr_b32 s51, s87, 16
-; SI-NEXT: s_mov_b32 s70, s23
-; SI-NEXT: s_mov_b32 s23, s24
-; SI-NEXT: v_readlane_b32 s24, v46, 45
-; SI-NEXT: s_mov_b32 s87, s25
-; SI-NEXT: s_mov_b32 s25, s26
-; SI-NEXT: s_mov_b32 s26, s27
-; SI-NEXT: s_mov_b32 s27, s28
-; SI-NEXT: s_mov_b32 s28, s92
-; SI-NEXT: s_lshr_b32 s53, s96, 16
-; SI-NEXT: s_mov_b32 s96, s90
-; SI-NEXT: s_mov_b32 s90, s79
-; SI-NEXT: s_mov_b32 s79, s31
-; SI-NEXT: s_mov_b32 s31, s35
-; SI-NEXT: s_mov_b32 s35, s37
-; SI-NEXT: s_mov_b32 s37, s19
-; SI-NEXT: s_mov_b32 s19, s20
-; SI-NEXT: v_readlane_b32 s20, v46, 51
-; SI-NEXT: s_lshr_b32 s92, s83, 16
-; SI-NEXT: s_mov_b32 s83, s16
-; SI-NEXT: v_readlane_b32 s16, v46, 49
-; SI-NEXT: s_cbranch_execnz .LBB97_3
-; SI-NEXT: .LBB97_2: ; %cmp.true
-; SI-NEXT: v_readlane_b32 s4, v46, 11
-; SI-NEXT: s_add_i32 s85, s4, 3
-; SI-NEXT: v_readlane_b32 s5, v46, 6
-; SI-NEXT: v_readlane_b32 s6, v47, 60
+; SI-NEXT: s_lshr_b32 s5, s5, 16
+; SI-NEXT: s_lshr_b32 s91, s91, 16
+; SI-NEXT: s_lshr_b32 s93, s93, 16
+; SI-NEXT: s_lshr_b32 s11, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s66, 16
+; SI-NEXT: s_lshr_b32 s7, s90, 16
+; SI-NEXT: s_lshr_b32 s9, s69, 16
+; SI-NEXT: s_lshr_b32 s39, s70, 16
+; SI-NEXT: s_mov_b32 s98, s27
+; SI-NEXT: s_mov_b32 s99, s25
+; SI-NEXT: v_readlane_b32 s70, v46, 56
+; SI-NEXT: v_readlane_b32 s27, v46, 55
+; SI-NEXT: s_lshr_b32 s49, s71, 16
+; SI-NEXT: s_lshr_b32 s51, vcc_hi, 16
+; SI-NEXT: s_mov_b32 s25, s24
+; SI-NEXT: s_mov_b32 s24, s68
+; SI-NEXT: s_mov_b32 s68, s26
+; SI-NEXT: s_mov_b32 s26, s28
+; SI-NEXT: s_mov_b32 s28, s29
+; SI-NEXT: s_mov_b32 s29, s67
+; SI-NEXT: v_readlane_b32 s69, v46, 54
+; SI-NEXT: v_readlane_b32 s90, v46, 53
+; SI-NEXT: s_lshr_b32 s53, s31, 16
+; SI-NEXT: s_lshr_b32 s55, s82, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s87, 16
+; SI-NEXT: s_mov_b64 s[66:67], 0
+; SI-NEXT: s_branch .LBB97_3
+; SI-NEXT: .LBB97_2:
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: .LBB97_3: ; %Flow
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: s_cselect_b32 s17, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s17, 1
+; SI-NEXT: s_cbranch_scc1 .LBB97_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s85, s99, 3
+; SI-NEXT: v_readlane_b32 s5, v46, 29
; SI-NEXT: s_and_b32 s4, s85, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 8
-; SI-NEXT: s_add_i32 s6, s6, 3
+; SI-NEXT: s_add_i32 s6, s22, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_readlane_b32 s5, v47, 58
+; SI-NEXT: v_readlane_b32 s5, v46, 20
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
@@ -193592,27 +194454,30 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s5, s17, 3
+; SI-NEXT: s_add_i32 s5, s19, 3
+; SI-NEXT: v_readlane_b32 s6, v47, 37
+; SI-NEXT: v_readlane_b32 s7, v47, 36
; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_lshl_b32 s6, s99, 8
-; SI-NEXT: s_add_i32 s7, s86, 3
+; SI-NEXT: s_lshl_b32 s6, s6, 8
+; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: v_readlane_b32 s6, v47, 35
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s83, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_addk_i32 s5, 0x300
; SI-NEXT: s_or_b32 s6, s6, s7
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_readlane_b32 s6, v46, 20
+; SI-NEXT: v_readlane_b32 s6, v46, 42
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: v_readlane_b32 s7, v46, 14
-; SI-NEXT: v_readlane_b32 s8, v46, 9
+; SI-NEXT: v_readlane_b32 s7, v46, 36
+; SI-NEXT: v_readlane_b32 s8, v46, 32
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 8
; SI-NEXT: s_add_i32 s83, s8, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v46, 8
+; SI-NEXT: v_readlane_b32 s7, v46, 31
; SI-NEXT: s_and_b32 s8, s83, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
@@ -193622,25 +194487,27 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s7, s29, 3
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s8, s24, 8
-; SI-NEXT: s_add_i32 s9, s19, 3
+; SI-NEXT: s_lshl_b32 s8, s68, 8
+; SI-NEXT: s_add_i32 s9, s24, 3
+; SI-NEXT: s_add_i32 s96, s4, 0x3000000
; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: v_readlane_b32 s4, v47, 38
; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: s_lshl_b32 s8, s16, 24
+; SI-NEXT: s_lshl_b32 s8, s4, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_addk_i32 s7, 0x300
; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: v_readlane_b32 s8, v46, 23
+; SI-NEXT: v_readlane_b32 s8, v46, 45
; SI-NEXT: s_add_i32 s8, s8, 3
-; SI-NEXT: v_readlane_b32 s9, v46, 22
-; SI-NEXT: v_readlane_b32 s10, v46, 19
+; SI-NEXT: v_readlane_b32 s9, v46, 44
+; SI-NEXT: v_readlane_b32 s10, v46, 41
; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 8
; SI-NEXT: s_add_i32 s68, s10, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readlane_b32 s9, v46, 16
+; SI-NEXT: v_readlane_b32 s9, v46, 38
; SI-NEXT: s_and_b32 s10, s68, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
@@ -193648,45 +194515,45 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s9, s9, s10
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: v_readlane_b32 s9, v46, 4
+; SI-NEXT: v_readlane_b32 s9, v46, 48
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s10, v46, 0
+; SI-NEXT: v_readlane_b32 s4, v47, 39
; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
+; SI-NEXT: s_lshl_b32 s10, s4, 8
; SI-NEXT: s_add_i32 s11, s28, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s11, s11, 0xff
-; SI-NEXT: s_lshl_b32 s10, s27, 24
+; SI-NEXT: s_lshl_b32 s10, s26, 24
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: s_addk_i32 s9, 0x300
; SI-NEXT: s_or_b32 s10, s10, s11
; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: v_readlane_b32 s10, v46, 25
+; SI-NEXT: v_readlane_b32 s10, v46, 47
; SI-NEXT: s_add_i32 s17, s10, 3
-; SI-NEXT: v_readlane_b32 s11, v46, 24
-; SI-NEXT: v_readlane_b32 s12, v46, 21
+; SI-NEXT: v_readlane_b32 s11, v46, 46
+; SI-NEXT: v_readlane_b32 s12, v46, 43
; SI-NEXT: s_and_b32 s10, s17, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 8
-; SI-NEXT: s_add_i32 s12, s12, 3
+; SI-NEXT: s_add_i32 s23, s12, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v46, 18
-; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: v_readlane_b32 s11, v46, 40
+; SI-NEXT: s_and_b32 s12, s23, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_addk_i32 s10, 0x300
; SI-NEXT: s_or_b32 s11, s11, s12
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v46, 10
+; SI-NEXT: v_readlane_b32 s11, v46, 33
; SI-NEXT: s_add_i32 s84, s11, 3
-; SI-NEXT: v_readlane_b32 s12, v46, 7
-; SI-NEXT: v_readlane_b32 s13, v46, 1
+; SI-NEXT: v_readlane_b32 s12, v46, 30
+; SI-NEXT: v_readlane_b32 s13, v46, 25
; SI-NEXT: s_and_b32 s11, s84, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_add_i32 s13, s13, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v47, 59
+; SI-NEXT: v_readlane_b32 s12, v46, 21
; SI-NEXT: s_and_b32 s13, s13, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
@@ -193694,15 +194561,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v46, 17
+; SI-NEXT: v_readlane_b32 s12, v46, 39
; SI-NEXT: s_add_i32 s99, s12, 3
-; SI-NEXT: v_readlane_b32 s13, v46, 15
-; SI-NEXT: v_readlane_b32 s14, v46, 12
+; SI-NEXT: v_readlane_b32 s13, v46, 37
+; SI-NEXT: v_readlane_b32 s14, v46, 34
; SI-NEXT: s_and_b32 s12, s99, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 8
; SI-NEXT: s_add_i32 s86, s14, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: v_readlane_b32 s13, v46, 13
+; SI-NEXT: v_readlane_b32 s13, v46, 35
; SI-NEXT: s_and_b32 s14, s86, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_lshl_b32 s14, s14, 16
@@ -193710,15 +194577,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s13, s13, s14
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: v_readlane_b32 s13, v46, 5
+; SI-NEXT: v_readlane_b32 s13, v46, 28
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: v_readlane_b32 s14, v46, 3
-; SI-NEXT: v_readlane_b32 s15, v47, 62
+; SI-NEXT: v_readlane_b32 s14, v46, 27
+; SI-NEXT: v_readlane_b32 s15, v46, 23
; SI-NEXT: s_and_b32 s13, s13, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_add_i32 s15, s15, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: v_readlane_b32 s14, v47, 63
+; SI-NEXT: v_readlane_b32 s14, v46, 24
; SI-NEXT: s_and_b32 s15, s15, 0xff
; SI-NEXT: s_lshl_b32 s14, s14, 24
; SI-NEXT: s_lshl_b32 s15, s15, 16
@@ -193726,41 +194593,41 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s14, s14, s15
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: v_readlane_b32 s14, v46, 2
+; SI-NEXT: v_readlane_b32 s14, v46, 26
; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: v_readlane_b32 s15, v47, 61
-; SI-NEXT: v_readlane_b32 s16, v47, 56
+; SI-NEXT: v_readlane_b32 s15, v46, 22
; SI-NEXT: s_and_b32 s14, s14, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s16, s69, 3
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: v_readlane_b32 s15, v47, 57
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s15, s15, 24
+; SI-NEXT: s_lshl_b32 s15, s90, 24
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_addk_i32 s14, 0x300
; SI-NEXT: s_or_b32 s15, s15, s16
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_add_i32 s15, s26, 3
+; SI-NEXT: v_readlane_b32 s15, v46, 19
+; SI-NEXT: s_add_i32 s15, s15, 3
+; SI-NEXT: v_readlane_b32 s4, v47, 40
; SI-NEXT: s_and_b32 s15, s15, 0xff
-; SI-NEXT: s_lshl_b32 s16, s25, 8
-; SI-NEXT: s_add_i32 s17, s23, 3
+; SI-NEXT: s_lshl_b32 s16, s98, 8
+; SI-NEXT: s_add_i32 s17, s4, 3
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s17, s17, 0xff
-; SI-NEXT: s_lshl_b32 s16, s87, 24
+; SI-NEXT: s_lshl_b32 s16, s25, 24
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_addk_i32 s15, 0x300
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
-; SI-NEXT: s_add_i32 s16, s20, 3
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v47, 54
+; SI-NEXT: s_add_i32 s16, s27, 3
+; SI-NEXT: v_readlane_b32 s18, v46, 17
; SI-NEXT: s_and_b32 s16, s16, 0xff
+; SI-NEXT: s_lshl_b32 s17, s70, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 55
+; SI-NEXT: v_readlane_b32 s17, v46, 18
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193769,66 +194636,83 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s40, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s22, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 15
+; SI-NEXT: v_readlane_b32 s18, v46, 12
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s65, 8
-; SI-NEXT: s_add_i32 s18, s94, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 13
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s67, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s41, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s71, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 14
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 11
+; SI-NEXT: v_readlane_b32 s18, v46, 9
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s81, 8
-; SI-NEXT: s_add_i32 s18, s55, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 10
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s97, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s42, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s98, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 8
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 7
+; SI-NEXT: v_readlane_b32 s18, v46, 3
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s78, 8
-; SI-NEXT: s_add_i32 s18, s31, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 4
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s35, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s43, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s82, 3
+; SI-NEXT: v_readlane_b32 s16, v46, 6
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v46, 5
+; SI-NEXT: v_readlane_b32 s18, v46, 1
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s37, 8
-; SI-NEXT: s_add_i32 s18, s90, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: v_readlane_b32 s17, v46, 2
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s79, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s44, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s96, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 46
-; SI-NEXT: v_readlane_b32 s18, v47, 45
+; SI-NEXT: v_readlane_b32 s16, v46, 0
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 63
+; SI-NEXT: v_readlane_b32 s18, v47, 59
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 44
+; SI-NEXT: v_readlane_b32 s17, v47, 60
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193837,12 +194721,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s45, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s21, 3
+; SI-NEXT: v_readlane_b32 s16, v47, 62
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 61
+; SI-NEXT: v_readlane_b32 s18, v47, 57
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s76, 8
-; SI-NEXT: s_add_i32 s18, s70, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 50
+; SI-NEXT: v_readlane_b32 s17, v47, 58
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193851,14 +194738,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s46, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s80, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 49
-; SI-NEXT: v_readlane_b32 s18, v47, 48
+; SI-NEXT: v_readlane_b32 s16, v47, 56
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 55
+; SI-NEXT: v_readlane_b32 s18, v47, 53
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 47
+; SI-NEXT: v_readlane_b32 s17, v47, 54
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193886,8 +194774,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_add_i32 s56, s16, 0x3000000
; SI-NEXT: v_readlane_b32 s16, v47, 14
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 53
-; SI-NEXT: v_readlane_b32 s18, v47, 52
+; SI-NEXT: v_readlane_b32 s17, v47, 52
+; SI-NEXT: v_readlane_b32 s18, v47, 50
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -193901,12 +194789,11 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s57, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 26
+; SI-NEXT: v_readlane_b32 s16, v47, 45
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 25
; SI-NEXT: v_readlane_b32 s18, v47, 24
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s92, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: v_readlane_b32 s17, v47, 23
@@ -193935,32 +194822,32 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s59, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 34
+; SI-NEXT: v_readlane_b32 s16, v47, 46
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 33
-; SI-NEXT: v_readlane_b32 s18, v47, 32
+; SI-NEXT: v_readlane_b32 s17, v47, 44
+; SI-NEXT: v_readlane_b32 s18, v47, 41
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 31
+; SI-NEXT: v_readlane_b32 s4, v47, 29
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s17, s4, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s60, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 30
+; SI-NEXT: v_readlane_b32 s16, v47, 28
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 29
-; SI-NEXT: v_readlane_b32 s18, v47, 28
+; SI-NEXT: v_readlane_b32 s17, v47, 27
+; SI-NEXT: v_readlane_b32 s18, v47, 26
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 27
+; SI-NEXT: v_readlane_b32 s17, v47, 25
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193969,14 +194856,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s61, s16, 0x3000000
-; SI-NEXT: s_add_i32 s16, s69, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 41
-; SI-NEXT: v_readlane_b32 s18, v47, 40
+; SI-NEXT: v_readlane_b32 s16, v47, 49
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s17, v47, 47
+; SI-NEXT: v_readlane_b32 s18, v47, 43
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 39
+; SI-NEXT: v_readlane_b32 s17, v47, 42
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -193985,15 +194873,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s62, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v47, 38
+; SI-NEXT: v_readlane_b32 s16, v47, 33
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v47, 37
-; SI-NEXT: v_readlane_b32 s18, v47, 36
+; SI-NEXT: v_readlane_b32 s17, v47, 32
+; SI-NEXT: v_readlane_b32 s18, v47, 31
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 35
+; SI-NEXT: v_readlane_b32 s17, v47, 30
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
@@ -194022,15 +194910,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_readlane_b32 s16, v47, 1
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: v_readlane_b32 s17, v47, 0
-; SI-NEXT: v_readlane_b32 s18, v47, 43
+; SI-NEXT: v_readlane_b32 s18, v47, 48
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v47, 42
+; SI-NEXT: v_readlane_b32 s4, v47, 34
; SI-NEXT: s_and_b32 s18, s18, 0xff
; SI-NEXT: s_addk_i32 s16, 0x300
-; SI-NEXT: s_lshl_b32 s17, s17, 24
+; SI-NEXT: s_lshl_b32 s17, s4, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
@@ -194068,191 +194956,154 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s75, s16, 0x3000000
-; SI-NEXT: s_lshr_b64 s[16:17], s[74:75], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 26
-; SI-NEXT: v_writelane_b32 v46, s17, 27
-; SI-NEXT: s_lshr_b64 s[16:17], s[72:73], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 28
-; SI-NEXT: v_writelane_b32 v46, s17, 29
-; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 30
-; SI-NEXT: v_writelane_b32 v46, s17, 31
-; SI-NEXT: s_lshr_b64 s[16:17], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 32
-; SI-NEXT: v_writelane_b32 v46, s17, 33
-; SI-NEXT: s_lshr_b64 s[16:17], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 34
-; SI-NEXT: v_writelane_b32 v46, s17, 35
-; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 36
-; SI-NEXT: v_writelane_b32 v46, s17, 37
-; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 38
-; SI-NEXT: v_writelane_b32 v46, s17, 39
-; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 16
-; SI-NEXT: s_add_i32 s10, s10, 0x3000000
-; SI-NEXT: s_add_i32 s11, s11, 0x3000000
+; SI-NEXT: s_add_i32 s34, s6, 0x3000000
+; SI-NEXT: s_add_i32 s35, s7, 0x3000000
+; SI-NEXT: s_add_i32 s36, s8, 0x3000000
+; SI-NEXT: s_add_i32 s37, s9, 0x3000000
+; SI-NEXT: s_add_i32 s94, s10, 0x3000000
+; SI-NEXT: s_add_i32 s95, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
; SI-NEXT: s_add_i32 s14, s14, 0x3000000
; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: v_writelane_b32 v46, s16, 40
-; SI-NEXT: s_add_i32 s4, s4, 0x3000000
-; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: s_add_i32 s8, s8, 0x3000000
-; SI-NEXT: s_add_i32 s9, s9, 0x3000000
-; SI-NEXT: v_writelane_b32 v46, s17, 41
-; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[50:51], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[52:53], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v46, s16, 42
-; SI-NEXT: s_lshr_b64 s[54:55], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[64:65], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[66:67], s[4:5], 16
-; SI-NEXT: s_lshr_b32 s77, s75, 16
-; SI-NEXT: s_lshr_b32 s68, s73, 16
-; SI-NEXT: s_lshr_b32 s88, s63, 16
-; SI-NEXT: s_lshr_b32 s84, s61, 16
-; SI-NEXT: s_lshr_b32 s89, s59, 16
-; SI-NEXT: s_lshr_b32 s91, s57, 16
-; SI-NEXT: s_lshr_b32 s93, s47, 16
-; SI-NEXT: s_lshr_b32 s95, s45, 16
-; SI-NEXT: s_lshr_b32 s30, s43, 16
-; SI-NEXT: s_lshr_b32 s34, s41, 16
-; SI-NEXT: s_lshr_b32 s36, s15, 16
-; SI-NEXT: s_lshr_b32 s39, s13, 16
-; SI-NEXT: s_lshr_b32 s49, s11, 16
-; SI-NEXT: s_lshr_b32 s51, s9, 16
-; SI-NEXT: s_lshr_b32 s53, s7, 16
-; SI-NEXT: s_lshr_b32 s92, s5, 16
-; SI-NEXT: v_writelane_b32 v46, s17, 43
-; SI-NEXT: .LBB97_3: ; %end
+; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s97, s5, 0x3000000
+; SI-NEXT: s_add_i32 s75, s16, 0x3000000
+; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[10:11], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[4:5], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[6:7], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[50:51], s[94:95], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[36:37], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[34:35], 16
+; SI-NEXT: s_lshr_b64 s[20:21], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[84:85], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[96:97], 16
+; SI-NEXT: s_lshr_b32 s23, s75, 16
+; SI-NEXT: s_lshr_b32 s77, s73, 16
+; SI-NEXT: s_lshr_b32 s5, s63, 16
+; SI-NEXT: s_lshr_b32 s91, s61, 16
+; SI-NEXT: s_lshr_b32 s93, s59, 16
+; SI-NEXT: s_lshr_b32 s11, s57, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s47, 16
+; SI-NEXT: s_lshr_b32 s30, s45, 16
+; SI-NEXT: s_lshr_b32 s7, s43, 16
+; SI-NEXT: s_lshr_b32 s9, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s15, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s51, s95, 16
+; SI-NEXT: s_lshr_b32 s53, s37, 16
+; SI-NEXT: s_lshr_b32 s55, s35, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s97, 16
+; SI-NEXT: .LBB97_5: ; %end
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s18, v46, 26
; SI-NEXT: s_and_b32 s17, s74, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v46, 27
-; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_lshl_b32 s18, s20, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s77, 16
-; SI-NEXT: v_readlane_b32 s20, v46, 28
+; SI-NEXT: s_lshl_b32 s19, s23, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s72, 0xffff
-; SI-NEXT: v_readlane_b32 s21, v46, 29
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_lshl_b32 s20, s76, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s68, 16
-; SI-NEXT: v_readlane_b32 s22, v46, 30
+; SI-NEXT: s_lshl_b32 s21, s77, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s62, 0xffff
-; SI-NEXT: v_readlane_b32 s23, v46, 31
-; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_lshl_b32 s22, s78, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s63, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s88, 16
-; SI-NEXT: v_readlane_b32 s24, v46, 32
+; SI-NEXT: s_lshl_b32 s23, s5, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s60, 0xffff
-; SI-NEXT: v_readlane_b32 s25, v46, 33
-; SI-NEXT: s_lshl_b32 s24, s24, 16
+; SI-NEXT: s_lshl_b32 s24, s80, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s61, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s84, 16
-; SI-NEXT: v_readlane_b32 s26, v46, 34
+; SI-NEXT: s_lshl_b32 s25, s91, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s58, 0xffff
-; SI-NEXT: v_readlane_b32 s27, v46, 35
-; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_lshl_b32 s26, s10, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s59, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s89, 16
-; SI-NEXT: v_readlane_b32 s28, v46, 36
+; SI-NEXT: s_lshl_b32 s27, s93, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s56, 0xffff
-; SI-NEXT: v_readlane_b32 s29, v46, 37
-; SI-NEXT: s_lshl_b32 s28, s28, 16
+; SI-NEXT: s_lshl_b32 s28, s4, 16
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s91, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 38
+; SI-NEXT: s_lshl_b32 s29, s11, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s46, 0xffff
-; SI-NEXT: v_readlane_b32 s57, v46, 39
-; SI-NEXT: s_lshl_b32 s46, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s6, 16
; SI-NEXT: s_or_b32 s29, s29, s46
; SI-NEXT: s_and_b32 s46, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s93, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 40
+; SI-NEXT: s_lshl_b32 s47, vcc_lo, 16
; SI-NEXT: s_or_b32 s46, s46, s47
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: v_readlane_b32 s57, v46, 41
-; SI-NEXT: s_lshl_b32 s47, s56, 16
+; SI-NEXT: s_lshl_b32 s47, s88, 16
; SI-NEXT: s_or_b32 s44, s44, s47
; SI-NEXT: s_and_b32 s45, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s95, 16
-; SI-NEXT: v_readlane_b32 s56, v46, 42
+; SI-NEXT: s_lshl_b32 s47, s30, 16
; SI-NEXT: s_or_b32 s45, s45, s47
; SI-NEXT: s_and_b32 s42, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s56, 16
+; SI-NEXT: s_lshl_b32 s47, s84, 16
; SI-NEXT: s_or_b32 s42, s42, s47
; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s30, 16
+; SI-NEXT: s_lshl_b32 s47, s7, 16
; SI-NEXT: s_or_b32 s43, s43, s47
; SI-NEXT: s_and_b32 s40, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s38, 16
+; SI-NEXT: s_lshl_b32 s47, s8, 16
; SI-NEXT: s_or_b32 s40, s40, s47
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s34, 16
+; SI-NEXT: s_lshl_b32 s47, s9, 16
; SI-NEXT: s_or_b32 s41, s41, s47
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s48, 16
+; SI-NEXT: s_lshl_b32 s47, s38, 16
; SI-NEXT: s_or_b32 s14, s14, s47
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s36, 16
+; SI-NEXT: s_lshl_b32 s47, s39, 16
; SI-NEXT: s_or_b32 s15, s15, s47
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s50, 16
+; SI-NEXT: s_lshl_b32 s47, s48, 16
; SI-NEXT: s_or_b32 s12, s12, s47
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s39, 16
+; SI-NEXT: s_lshl_b32 s47, s49, 16
; SI-NEXT: s_or_b32 s13, s13, s47
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s52, 16
+; SI-NEXT: s_and_b32 s10, s94, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s50, 16
; SI-NEXT: s_or_b32 s10, s10, s47
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s49, 16
+; SI-NEXT: s_and_b32 s11, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s51, 16
; SI-NEXT: s_or_b32 s11, s11, s47
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s54, 16
+; SI-NEXT: s_and_b32 s8, s36, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s52, 16
; SI-NEXT: s_or_b32 s8, s8, s47
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s51, 16
+; SI-NEXT: s_and_b32 s9, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s53, 16
; SI-NEXT: s_or_b32 s9, s9, s47
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s64, 16
+; SI-NEXT: s_and_b32 s6, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s54, 16
; SI-NEXT: s_or_b32 s6, s6, s47
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s53, 16
+; SI-NEXT: s_and_b32 s7, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s55, 16
; SI-NEXT: s_or_b32 s7, s7, s47
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s47, s66, 16
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s92, 16
+; SI-NEXT: s_and_b32 s4, s96, 0xffff
+; SI-NEXT: s_lshl_b32 s47, s64, 16
+; SI-NEXT: s_and_b32 s5, s97, 0xffff
+; SI-NEXT: s_lshl_b32 s16, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s47
; SI-NEXT: s_or_b32 s5, s5, s16
; SI-NEXT: v_readlane_b32 s30, v45, 34
-; SI-NEXT: v_readlane_b32 s57, v46, 43
; SI-NEXT: v_mov_b32_e32 v0, s17
; SI-NEXT: v_mov_b32_e32 v1, s18
; SI-NEXT: v_mov_b32_e32 v2, s19
@@ -194327,74 +195178,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB97_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 26
-; SI-NEXT: v_writelane_b32 v46, s5, 27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: v_writelane_b32 v46, s4, 28
-; SI-NEXT: v_writelane_b32 v46, s5, 29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 30
-; SI-NEXT: v_writelane_b32 v46, s5, 31
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 32
-; SI-NEXT: v_writelane_b32 v46, s5, 33
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 34
-; SI-NEXT: v_writelane_b32 v46, s5, 35
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 36
-; SI-NEXT: v_writelane_b32 v46, s5, 37
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 38
-; SI-NEXT: v_writelane_b32 v46, s5, 39
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 40
-; SI-NEXT: v_writelane_b32 v46, s5, 41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v46, s4, 42
-; SI-NEXT: v_writelane_b32 v46, s5, 43
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB97_2
;
; VI-LABEL: bitcast_v128i8_to_v64i16_scalar:
; VI: ; %bb.0:
@@ -194611,7 +195394,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; VI-NEXT: s_cbranch_scc0 .LBB97_4
+; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
@@ -194831,12 +195614,32 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v35, v43
; VI-NEXT: v_perm_b32 v43, v44, v43, s4
; VI-NEXT: v_or_b32_e32 v31, v43, v31
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: v_mov_b32_e32 v43, v39
; VI-NEXT: v_mov_b32_e32 v39, v34
; VI-NEXT: v_mov_b32_e32 v34, v52
; VI-NEXT: v_mov_b32_e32 v52, v45
-; VI-NEXT: s_cbranch_execnz .LBB97_3
-; VI-NEXT: .LBB97_2: ; %cmp.true
+; VI-NEXT: s_branch .LBB97_3
+; VI-NEXT: .LBB97_2:
+; VI-NEXT: v_mov_b32_e32 v43, v39
+; VI-NEXT: v_mov_b32_e32 v52, v34
+; VI-NEXT: v_mov_b32_e32 v39, v33
+; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v34, v53
+; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v62, v2
+; VI-NEXT: v_mov_b32_e32 v35, v1
+; VI-NEXT: v_mov_b32_e32 v55, v57
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; VI-NEXT: .LBB97_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB97_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -195251,7 +196054,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_or_b32_sdwa v27, v46, v27 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v18, vcc, 0x3000000, v18
; VI-NEXT: v_add_u32_e32 v27, vcc, 0x3000000, v27
-; VI-NEXT: .LBB97_3: ; %end
+; VI-NEXT: .LBB97_5: ; %end
; VI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
@@ -195270,20 +196073,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB97_4:
-; VI-NEXT: v_mov_b32_e32 v43, v39
-; VI-NEXT: v_mov_b32_e32 v52, v34
-; VI-NEXT: v_mov_b32_e32 v39, v33
-; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v34, v53
-; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v62, v2
-; VI-NEXT: v_mov_b32_e32 v35, v1
-; VI-NEXT: v_mov_b32_e32 v55, v57
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; VI-NEXT: s_branch .LBB97_2
;
; GFX9-LABEL: bitcast_v128i8_to_v64i16_scalar:
; GFX9: ; %bb.0:
@@ -195775,8 +196564,10 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX9-NEXT: .LBB97_3: ; %Flow
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB97_5
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v48
@@ -196328,7 +197119,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v86
@@ -196480,9 +197271,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v183.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v41.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB97_3
-; GFX11-TRUE16-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB97_3
+; GFX11-TRUE16-NEXT: .LBB97_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
@@ -196711,16 +197511,12 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
-; GFX11-TRUE16-NEXT: .LBB97_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:324
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB97_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-TRUE16-NEXT: s_branch .LBB97_2
;
; GFX11-FAKE16-LABEL: bitcast_v128i8_to_v64i16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -196844,7 +197640,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s74, 0xff
@@ -197035,9 +197831,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v40
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB97_3
-; GFX11-FAKE16-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB97_3
+; GFX11-FAKE16-NEXT: .LBB97_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-FAKE16-NEXT: .LBB97_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v182
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v180
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v178
@@ -197298,14 +198103,10 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v29, v150, 16, v29
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v30, v30, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v31, v31, 16, v35
-; GFX11-FAKE16-NEXT: .LBB97_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB97_5: ; %end
; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s32 offset:320 ; 4-byte Folded Reload
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB97_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; GFX11-FAKE16-NEXT: s_branch .LBB97_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -202227,9 +203028,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
-; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: v_writelane_b32 v20, s34, 0
; SI-NEXT: v_writelane_b32 v20, s35, 1
; SI-NEXT: v_writelane_b32 v20, s36, 2
@@ -202266,799 +203066,1037 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v20, s99, 33
; SI-NEXT: v_writelane_b32 v20, s30, 34
; SI-NEXT: v_writelane_b32 v20, s31, 35
-; SI-NEXT: s_lshr_b32 s5, s16, 16
-; SI-NEXT: v_readfirstlane_b32 s49, v18
-; SI-NEXT: v_readfirstlane_b32 s51, v17
-; SI-NEXT: v_readfirstlane_b32 s38, v16
-; SI-NEXT: v_readfirstlane_b32 s48, v15
-; SI-NEXT: v_readfirstlane_b32 s37, v14
-; SI-NEXT: v_readfirstlane_b32 s95, v13
-; SI-NEXT: v_readfirstlane_b32 s96, v12
-; SI-NEXT: v_readfirstlane_b32 s36, v11
-; SI-NEXT: v_readfirstlane_b32 s85, v10
-; SI-NEXT: v_readfirstlane_b32 s87, v9
-; SI-NEXT: v_readfirstlane_b32 s34, v8
-; SI-NEXT: v_readfirstlane_b32 s84, v7
-; SI-NEXT: v_readfirstlane_b32 s71, v6
-; SI-NEXT: v_readfirstlane_b32 s81, v5
-; SI-NEXT: v_readfirstlane_b32 s68, v4
-; SI-NEXT: v_readfirstlane_b32 s70, v3
-; SI-NEXT: v_readfirstlane_b32 s67, v2
-; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: v_readfirstlane_b32 s30, v8
+; SI-NEXT: s_lshr_b32 s7, s30, 16
+; SI-NEXT: v_readfirstlane_b32 s69, v6
; SI-NEXT: ; implicit-def: $vgpr21 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 s75, s29, 16
-; SI-NEXT: s_lshr_b32 s66, s28, 16
-; SI-NEXT: s_lshr_b32 s78, s27, 16
-; SI-NEXT: s_lshr_b32 s55, s26, 16
-; SI-NEXT: s_lshr_b32 s79, s25, 16
-; SI-NEXT: s_lshr_b32 s54, s24, 16
-; SI-NEXT: s_lshr_b32 s90, s23, 16
-; SI-NEXT: s_lshr_b32 s7, s22, 16
-; SI-NEXT: s_lshr_b32 s91, s21, 16
-; SI-NEXT: s_lshr_b32 s53, s20, 16
-; SI-NEXT: s_lshr_b32 s92, s19, 16
-; SI-NEXT: s_lshr_b32 s6, s18, 16
-; SI-NEXT: s_lshr_b32 s93, s17, 16
-; SI-NEXT: s_lshr_b32 s58, s49, 16
-; SI-NEXT: s_lshr_b32 s50, s51, 16
-; SI-NEXT: s_lshr_b32 s59, s38, 16
-; SI-NEXT: s_lshr_b32 s39, s48, 16
-; SI-NEXT: s_lshr_b32 s60, s37, 16
-; SI-NEXT: s_lshr_b32 s94, s95, 16
-; SI-NEXT: s_lshr_b32 s61, s96, 16
-; SI-NEXT: s_lshr_b32 s97, s36, 16
-; SI-NEXT: s_lshr_b32 s62, s85, 16
-; SI-NEXT: s_lshr_b32 s86, s87, 16
-; SI-NEXT: s_lshr_b32 s63, s34, 16
-; SI-NEXT: s_lshr_b32 s35, s84, 16
-; SI-NEXT: s_lshr_b32 s72, s71, 16
-; SI-NEXT: s_lshr_b32 s80, s81, 16
-; SI-NEXT: s_lshr_b32 s73, s68, 16
-; SI-NEXT: s_lshr_b32 s69, s70, 16
-; SI-NEXT: s_lshr_b32 s74, s67, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
+; SI-NEXT: s_lshr_b32 s6, s69, 16
+; SI-NEXT: v_readfirstlane_b32 s66, v4
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_writelane_b32 v21, s7, 0
+; SI-NEXT: s_lshr_b32 s9, s66, 16
+; SI-NEXT: v_readfirstlane_b32 s55, v2
+; SI-NEXT: v_writelane_b32 v21, s6, 1
+; SI-NEXT: s_lshr_b32 s8, s55, 16
+; SI-NEXT: v_writelane_b32 v21, s9, 2
+; SI-NEXT: s_lshr_b32 s11, s29, 16
+; SI-NEXT: v_writelane_b32 v21, s8, 3
+; SI-NEXT: s_lshr_b32 s10, s27, 16
+; SI-NEXT: v_writelane_b32 v21, s11, 4
+; SI-NEXT: s_lshr_b32 s13, s25, 16
+; SI-NEXT: v_writelane_b32 v21, s10, 5
+; SI-NEXT: s_lshr_b32 s12, s23, 16
+; SI-NEXT: v_writelane_b32 v21, s13, 6
+; SI-NEXT: s_lshr_b32 s15, s21, 16
+; SI-NEXT: v_writelane_b32 v21, s12, 7
+; SI-NEXT: s_lshr_b32 s14, s19, 16
+; SI-NEXT: v_writelane_b32 v21, s15, 8
+; SI-NEXT: s_lshr_b32 s5, s16, 16
+; SI-NEXT: v_writelane_b32 v21, s14, 9
+; SI-NEXT: s_lshr_b32 s40, s18, 16
+; SI-NEXT: v_writelane_b32 v21, s5, 10
+; SI-NEXT: s_lshr_b32 s41, s20, 16
+; SI-NEXT: v_writelane_b32 v21, s40, 11
+; SI-NEXT: s_lshr_b32 s42, s22, 16
+; SI-NEXT: v_writelane_b32 v21, s41, 12
+; SI-NEXT: s_lshr_b32 s43, s24, 16
+; SI-NEXT: v_writelane_b32 v21, s42, 13
+; SI-NEXT: s_lshr_b32 s44, s26, 16
+; SI-NEXT: v_writelane_b32 v21, s43, 14
+; SI-NEXT: s_lshr_b32 s45, s28, 16
+; SI-NEXT: v_readfirstlane_b32 s95, v1
+; SI-NEXT: v_writelane_b32 v21, s44, 15
+; SI-NEXT: v_readfirstlane_b32 s68, v3
+; SI-NEXT: s_lshr_b32 s60, s95, 16
+; SI-NEXT: v_writelane_b32 v21, s45, 16
+; SI-NEXT: v_readfirstlane_b32 s71, v5
+; SI-NEXT: s_lshr_b32 s62, s68, 16
+; SI-NEXT: v_writelane_b32 v21, s60, 17
+; SI-NEXT: v_readfirstlane_b32 s80, v7
+; SI-NEXT: s_lshr_b32 s63, s71, 16
+; SI-NEXT: v_writelane_b32 v21, s62, 18
+; SI-NEXT: v_readfirstlane_b32 s85, v9
+; SI-NEXT: s_lshr_b32 s72, s80, 16
+; SI-NEXT: v_writelane_b32 v21, s63, 19
+; SI-NEXT: v_readfirstlane_b32 s34, v11
+; SI-NEXT: s_lshr_b32 s73, s85, 16
+; SI-NEXT: v_writelane_b32 v21, s72, 20
+; SI-NEXT: v_readfirstlane_b32 s93, v13
+; SI-NEXT: s_lshr_b32 s74, s34, 16
+; SI-NEXT: v_writelane_b32 v21, s73, 21
+; SI-NEXT: v_readfirstlane_b32 s39, v18
+; SI-NEXT: v_readfirstlane_b32 s49, v17
+; SI-NEXT: v_readfirstlane_b32 s36, v16
+; SI-NEXT: v_readfirstlane_b32 s38, v15
+; SI-NEXT: v_readfirstlane_b32 s35, v14
+; SI-NEXT: s_lshr_b32 s78, s93, 16
+; SI-NEXT: v_readfirstlane_b32 s86, v12
+; SI-NEXT: v_readfirstlane_b32 s81, v10
+; SI-NEXT: v_writelane_b32 v21, s74, 22
+; SI-NEXT: s_lshr_b32 s91, s17, 16
+; SI-NEXT: s_lshr_b32 s59, s39, 16
+; SI-NEXT: s_lshr_b32 s92, s49, 16
+; SI-NEXT: s_lshr_b32 s51, s36, 16
+; SI-NEXT: s_lshr_b32 s90, s38, 16
+; SI-NEXT: s_lshr_b32 s61, s35, 16
+; SI-NEXT: s_lshr_b32 s53, s86, 16
+; SI-NEXT: s_lshr_b32 s58, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v19
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_writelane_b32 v21, s5, 0
+; SI-NEXT: v_writelane_b32 v21, s78, 23
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v21, s6, 1
-; SI-NEXT: v_writelane_b32 v21, s7, 2
+; SI-NEXT: v_writelane_b32 v21, s90, 24
+; SI-NEXT: v_writelane_b32 v21, s92, 25
; SI-NEXT: ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
-; SI-NEXT: ; implicit-def: $vgpr23 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB99_4
+; SI-NEXT: s_cbranch_scc0 .LBB99_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_or_b32 s98, s4, s5
+; SI-NEXT: s_or_b32 vcc_lo, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s93, 16
-; SI-NEXT: s_or_b32 s99, s4, s5
+; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: s_or_b32 vcc_hi, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s6, 16
+; SI-NEXT: s_lshl_b32 s5, s40, 16
; SI-NEXT: s_or_b32 s82, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_lshl_b32 s5, s14, 16
; SI-NEXT: s_or_b32 s83, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s53, 16
+; SI-NEXT: s_lshl_b32 s5, s41, 16
; SI-NEXT: s_or_b32 s64, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: s_lshl_b32 s5, s15, 16
; SI-NEXT: s_or_b32 s65, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_lshl_b32 s5, s42, 16
; SI-NEXT: s_or_b32 s88, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_lshl_b32 s5, s12, 16
; SI-NEXT: s_or_b32 s89, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s54, 16
+; SI-NEXT: s_lshl_b32 s5, s43, 16
; SI-NEXT: s_or_b32 s56, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s79, 16
+; SI-NEXT: s_lshl_b32 s5, s13, 16
; SI-NEXT: s_or_b32 s57, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: s_lshl_b32 s5, s44, 16
; SI-NEXT: s_or_b32 s76, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_lshl_b32 s5, s10, 16
; SI-NEXT: s_or_b32 s77, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_lshl_b32 s5, s45, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s75, 16
+; SI-NEXT: s_lshl_b32 s5, s11, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s60, 16
; SI-NEXT: s_or_b32 s44, s4, s5
-; SI-NEXT: s_and_b32 s4, s67, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_and_b32 s4, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
-; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s73, 16
+; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_or_b32 s42, s4, s5
+; SI-NEXT: s_and_b32 s4, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s9, 16
; SI-NEXT: s_or_b32 s43, s4, s5
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
-; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_or_b32 s40, s4, s5
+; SI-NEXT: s_and_b32 s4, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s41, s4, s5
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s72, 16
+; SI-NEXT: s_mov_b32 s79, s14
; SI-NEXT: s_or_b32 s14, s4, s5
-; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s63, 16
+; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s7, 16
+; SI-NEXT: s_mov_b32 s50, s15
; SI-NEXT: s_or_b32 s15, s4, s5
-; SI-NEXT: s_and_b32 s4, s87, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
-; SI-NEXT: s_or_b32 s12, s4, s5
; SI-NEXT: s_and_b32 s4, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s62, 16
+; SI-NEXT: s_lshl_b32 s5, s73, 16
+; SI-NEXT: s_mov_b32 s75, s12
+; SI-NEXT: s_or_b32 s12, s4, s5
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s58, 16
+; SI-NEXT: s_mov_b32 s37, s13
; SI-NEXT: s_or_b32 s13, s4, s5
-; SI-NEXT: s_and_b32 s4, s36, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_and_b32 s4, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s74, 16
+; SI-NEXT: s_mov_b32 s73, s10
; SI-NEXT: s_or_b32 s10, s4, s5
-; SI-NEXT: s_and_b32 s4, s96, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_and_b32 s4, s86, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s53, 16
+; SI-NEXT: s_mov_b32 s72, s11
; SI-NEXT: s_or_b32 s11, s4, s5
-; SI-NEXT: s_and_b32 s4, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_and_b32 s4, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
+; SI-NEXT: s_mov_b32 s63, s8
; SI-NEXT: s_or_b32 s8, s4, s5
-; SI-NEXT: s_and_b32 s4, s37, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s60, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s61, 16
+; SI-NEXT: s_mov_b32 s62, s9
; SI-NEXT: s_or_b32 s9, s4, s5
-; SI-NEXT: s_and_b32 s4, s48, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s39, 16
-; SI-NEXT: s_or_b32 s6, s4, s5
; SI-NEXT: s_and_b32 s4, s38, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s59, 16
+; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_mov_b32 s94, s6
+; SI-NEXT: s_or_b32 s6, s4, s5
+; SI-NEXT: s_and_b32 s4, s36, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s51, 16
+; SI-NEXT: s_mov_b32 s60, s7
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_and_b32 s4, s51, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
+; SI-NEXT: s_and_b32 s4, s49, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s49, 0xffff
-; SI-NEXT: s_lshl_b32 vcc_lo, s58, 16
-; SI-NEXT: s_or_b32 s5, s5, vcc_lo
-; SI-NEXT: s_lshr_b32 vcc_lo, s99, 8
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 36
-; SI-NEXT: s_lshr_b32 vcc_lo, s83, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 38
-; SI-NEXT: s_lshr_b32 vcc_lo, s65, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 40
-; SI-NEXT: s_lshr_b32 vcc_lo, s89, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 42
-; SI-NEXT: s_lshr_b32 vcc_lo, s57, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 44
-; SI-NEXT: s_lshr_b32 vcc_lo, s77, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 46
-; SI-NEXT: s_lshr_b32 vcc_lo, s47, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 48
-; SI-NEXT: s_lshr_b32 vcc_lo, s45, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 50
-; SI-NEXT: s_lshr_b32 vcc_lo, s43, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 52
-; SI-NEXT: s_lshr_b32 vcc_lo, s41, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 54
-; SI-NEXT: s_lshr_b32 vcc_lo, s15, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 56
-; SI-NEXT: s_lshr_b32 vcc_lo, s13, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 58
-; SI-NEXT: s_lshr_b32 vcc_lo, s11, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 60
-; SI-NEXT: s_lshr_b32 vcc_lo, s9, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 62
-; SI-NEXT: s_lshr_b32 vcc_lo, s7, 8
+; SI-NEXT: s_and_b32 s5, s39, 0xffff
+; SI-NEXT: s_lshl_b32 s96, s59, 16
+; SI-NEXT: s_or_b32 s5, s5, s96
+; SI-NEXT: s_lshr_b64 s[96:97], vcc, 24
+; SI-NEXT: v_writelane_b32 v21, s96, 30
+; SI-NEXT: v_writelane_b32 v21, s97, 31
+; SI-NEXT: s_lshr_b64 s[96:97], vcc, 16
+; SI-NEXT: v_writelane_b32 v21, s96, 28
+; SI-NEXT: v_writelane_b32 v21, s97, 29
+; SI-NEXT: s_lshr_b64 s[96:97], vcc, 8
+; SI-NEXT: v_writelane_b32 v21, s96, 26
+; SI-NEXT: v_writelane_b32 v21, s97, 27
+; SI-NEXT: s_lshr_b64 s[96:97], s[82:83], 24
+; SI-NEXT: v_writelane_b32 v21, s96, 36
+; SI-NEXT: v_writelane_b32 v21, s97, 37
+; SI-NEXT: s_lshr_b64 s[96:97], s[82:83], 16
+; SI-NEXT: v_writelane_b32 v21, s96, 34
+; SI-NEXT: v_writelane_b32 v21, s97, 35
+; SI-NEXT: s_lshr_b64 s[96:97], s[82:83], 8
+; SI-NEXT: v_writelane_b32 v21, s96, 32
+; SI-NEXT: v_writelane_b32 v21, s97, 33
+; SI-NEXT: s_lshr_b64 s[96:97], s[64:65], 24
+; SI-NEXT: v_writelane_b32 v21, s96, 42
+; SI-NEXT: v_writelane_b32 v21, s97, 43
+; SI-NEXT: s_lshr_b64 s[96:97], s[64:65], 16
+; SI-NEXT: v_writelane_b32 v21, s96, 40
+; SI-NEXT: v_writelane_b32 v21, s97, 41
+; SI-NEXT: s_lshr_b64 s[96:97], s[64:65], 8
+; SI-NEXT: v_writelane_b32 v21, s96, 38
+; SI-NEXT: v_writelane_b32 v21, s97, 39
+; SI-NEXT: s_lshr_b64 s[96:97], s[88:89], 24
+; SI-NEXT: v_writelane_b32 v21, s96, 48
+; SI-NEXT: v_writelane_b32 v21, s97, 49
+; SI-NEXT: s_lshr_b64 s[96:97], s[88:89], 16
+; SI-NEXT: v_writelane_b32 v21, s96, 46
+; SI-NEXT: v_writelane_b32 v21, s97, 47
+; SI-NEXT: s_lshr_b64 s[96:97], s[88:89], 8
+; SI-NEXT: v_writelane_b32 v21, s96, 44
+; SI-NEXT: s_lshr_b32 s74, vcc_hi, 8
+; SI-NEXT: v_writelane_b32 v21, s97, 45
+; SI-NEXT: s_lshr_b64 s[96:97], s[56:57], 24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v22, s74, 53
+; SI-NEXT: s_lshr_b32 s74, s83, 8
+; SI-NEXT: v_writelane_b32 v21, s96, 54
+; SI-NEXT: v_writelane_b32 v22, s74, 55
+; SI-NEXT: s_lshr_b32 s74, s65, 8
+; SI-NEXT: v_writelane_b32 v21, s97, 55
+; SI-NEXT: s_lshr_b64 s[96:97], s[56:57], 16
+; SI-NEXT: v_writelane_b32 v22, s74, 57
+; SI-NEXT: s_lshr_b32 s74, s89, 8
+; SI-NEXT: v_writelane_b32 v21, s96, 52
+; SI-NEXT: v_writelane_b32 v22, s74, 59
+; SI-NEXT: s_lshr_b32 s74, s57, 8
+; SI-NEXT: v_writelane_b32 v21, s97, 53
+; SI-NEXT: s_lshr_b64 s[96:97], s[56:57], 8
+; SI-NEXT: v_writelane_b32 v22, s74, 61
+; SI-NEXT: s_lshr_b32 s74, s77, 8
+; SI-NEXT: v_writelane_b32 v21, s96, 50
+; SI-NEXT: v_writelane_b32 v22, s74, 50
+; SI-NEXT: s_bfe_u32 s52, s91, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s97, 51
+; SI-NEXT: s_lshr_b64 s[96:97], s[76:77], 24
+; SI-NEXT: v_writelane_b32 v22, s52, 52
+; SI-NEXT: s_bfe_u32 s79, s79, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s96, 60
+; SI-NEXT: v_writelane_b32 v22, s79, 54
+; SI-NEXT: s_bfe_u32 s79, s50, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s97, 61
+; SI-NEXT: s_lshr_b64 s[96:97], s[76:77], 16
+; SI-NEXT: v_writelane_b32 v22, s79, 56
+; SI-NEXT: s_bfe_u32 s75, s75, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s96, 58
+; SI-NEXT: v_writelane_b32 v22, s75, 58
+; SI-NEXT: s_bfe_u32 s75, s37, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s97, 59
+; SI-NEXT: s_lshr_b64 s[96:97], s[76:77], 8
+; SI-NEXT: v_writelane_b32 v22, s75, 60
+; SI-NEXT: s_bfe_u32 s73, s73, 0x80008
+; SI-NEXT: v_writelane_b32 v21, s96, 56
+; SI-NEXT: v_writelane_b32 v22, s73, 51
+; SI-NEXT: v_writelane_b32 v21, s97, 57
+; SI-NEXT: s_lshr_b64 s[96:97], s[46:47], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 2
+; SI-NEXT: v_writelane_b32 v22, s97, 3
+; SI-NEXT: s_lshr_b64 s[96:97], s[46:47], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 0
+; SI-NEXT: v_writelane_b32 v22, s97, 1
+; SI-NEXT: s_lshr_b64 s[96:97], s[46:47], 8
+; SI-NEXT: v_writelane_b32 v21, s96, 62
+; SI-NEXT: v_writelane_b32 v21, s97, 63
+; SI-NEXT: s_lshr_b64 s[96:97], s[44:45], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 8
+; SI-NEXT: v_writelane_b32 v22, s97, 9
+; SI-NEXT: s_lshr_b64 s[96:97], s[44:45], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 6
+; SI-NEXT: v_writelane_b32 v22, s97, 7
+; SI-NEXT: s_lshr_b64 s[96:97], s[44:45], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 4
+; SI-NEXT: v_writelane_b32 v22, s97, 5
+; SI-NEXT: s_lshr_b64 s[96:97], s[42:43], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 14
+; SI-NEXT: v_writelane_b32 v22, s97, 15
+; SI-NEXT: s_lshr_b64 s[96:97], s[42:43], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 12
+; SI-NEXT: v_writelane_b32 v22, s97, 13
+; SI-NEXT: s_lshr_b64 s[96:97], s[42:43], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 10
+; SI-NEXT: v_writelane_b32 v22, s97, 11
+; SI-NEXT: s_lshr_b64 s[96:97], s[40:41], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 20
+; SI-NEXT: v_writelane_b32 v22, s97, 21
+; SI-NEXT: s_lshr_b64 s[96:97], s[40:41], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 18
+; SI-NEXT: v_writelane_b32 v22, s97, 19
+; SI-NEXT: s_lshr_b64 s[96:97], s[40:41], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 16
+; SI-NEXT: v_writelane_b32 v22, s97, 17
+; SI-NEXT: s_lshr_b64 s[96:97], s[14:15], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 26
+; SI-NEXT: v_writelane_b32 v22, s97, 27
+; SI-NEXT: s_lshr_b64 s[96:97], s[14:15], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 24
+; SI-NEXT: v_writelane_b32 v22, s97, 25
+; SI-NEXT: s_lshr_b64 s[96:97], s[14:15], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 22
+; SI-NEXT: v_writelane_b32 v22, s97, 23
+; SI-NEXT: s_lshr_b64 s[96:97], s[12:13], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 32
+; SI-NEXT: v_writelane_b32 v22, s97, 33
+; SI-NEXT: s_lshr_b64 s[96:97], s[12:13], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 30
+; SI-NEXT: v_writelane_b32 v22, s97, 31
+; SI-NEXT: s_lshr_b64 s[96:97], s[12:13], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 28
+; SI-NEXT: v_writelane_b32 v22, s97, 29
+; SI-NEXT: s_lshr_b64 s[96:97], s[10:11], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 38
+; SI-NEXT: v_writelane_b32 v22, s97, 39
+; SI-NEXT: s_lshr_b64 s[96:97], s[10:11], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 36
+; SI-NEXT: v_writelane_b32 v22, s97, 37
+; SI-NEXT: s_lshr_b64 s[96:97], s[10:11], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 34
+; SI-NEXT: v_writelane_b32 v22, s97, 35
+; SI-NEXT: s_lshr_b64 s[96:97], s[8:9], 24
+; SI-NEXT: v_writelane_b32 v22, s96, 44
+; SI-NEXT: v_writelane_b32 v22, s97, 45
+; SI-NEXT: s_lshr_b64 s[96:97], s[8:9], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 42
+; SI-NEXT: v_writelane_b32 v22, s97, 43
+; SI-NEXT: s_lshr_b64 s[96:97], s[8:9], 8
+; SI-NEXT: v_writelane_b32 v22, s96, 40
+; SI-NEXT: v_writelane_b32 v22, s97, 41
+; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 16
+; SI-NEXT: v_writelane_b32 v22, s96, 48
+; SI-NEXT: s_bfe_u32 s37, s94, 0x80008
+; SI-NEXT: s_mov_b32 s94, s58
+; SI-NEXT: s_bfe_u32 s84, s58, 0x80008
+; SI-NEXT: s_bfe_u32 s98, s59, 0x80008
+; SI-NEXT: s_mov_b32 s50, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[6:7], 24
+; SI-NEXT: v_writelane_b32 v22, s97, 49
+; SI-NEXT: s_lshr_b64 s[96:97], s[6:7], 8
+; SI-NEXT: s_bfe_u32 s54, s51, 0x80008
+; SI-NEXT: s_mov_b32 s59, s50
+; SI-NEXT: v_writelane_b32 v22, s96, 46
+; SI-NEXT: s_mov_b32 s52, s51
+; SI-NEXT: s_lshr_b64 s[50:51], s[4:5], 24
+; SI-NEXT: s_bfe_u32 s75, s63, 0x80008
+; SI-NEXT: s_bfe_u32 s63, s53, 0x80008
+; SI-NEXT: v_writelane_b32 v22, s97, 47
+; SI-NEXT: s_mov_b32 s51, s52
+; SI-NEXT: s_mov_b32 s96, s53
+; SI-NEXT: s_lshr_b64 s[52:53], s[4:5], 16
+; SI-NEXT: s_bfe_u32 s79, s72, 0x80008
+; SI-NEXT: s_bfe_u32 s73, s62, 0x80008
+; SI-NEXT: s_bfe_u32 s72, s60, 0x80008
+; SI-NEXT: s_bfe_u32 s62, s61, 0x80008
+; SI-NEXT: s_mov_b32 s53, s96
+; SI-NEXT: s_mov_b32 s96, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[4:5], 8
+; SI-NEXT: s_lshr_b32 s90, s47, 8
+; SI-NEXT: s_lshr_b32 s78, s45, 8
+; SI-NEXT: s_lshr_b32 s74, s43, 8
+; SI-NEXT: s_lshr_b32 s48, s41, 8
+; SI-NEXT: s_lshr_b32 s92, s15, 8
+; SI-NEXT: s_lshr_b32 s87, s13, 8
+; SI-NEXT: s_lshr_b32 s31, s11, 8
+; SI-NEXT: s_lshr_b32 s70, s9, 8
+; SI-NEXT: s_lshr_b32 s67, s7, 8
+; SI-NEXT: s_lshr_b32 s99, s5, 8
+; SI-NEXT: s_mov_b32 s61, s96
+; SI-NEXT: s_mov_b64 s[96:97], 0
+; SI-NEXT: s_branch .LBB99_3
+; SI-NEXT: .LBB99_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b32 s94, s58
+; SI-NEXT: v_writelane_b32 v21, s4, 26
+; SI-NEXT: v_writelane_b32 v21, s5, 27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b64 s[96:97], -1
+; SI-NEXT: v_writelane_b32 v21, s4, 28
+; SI-NEXT: v_writelane_b32 v21, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: v_writelane_b32 v21, s4, 30
+; SI-NEXT: v_writelane_b32 v21, s5, 31
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 32
+; SI-NEXT: v_writelane_b32 v21, s5, 33
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 34
+; SI-NEXT: v_writelane_b32 v21, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 36
+; SI-NEXT: v_writelane_b32 v21, s5, 37
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 38
+; SI-NEXT: v_writelane_b32 v21, s5, 39
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 40
+; SI-NEXT: v_writelane_b32 v21, s5, 41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 42
+; SI-NEXT: v_writelane_b32 v21, s5, 43
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 44
+; SI-NEXT: v_writelane_b32 v21, s5, 45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 46
+; SI-NEXT: v_writelane_b32 v21, s5, 47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 48
+; SI-NEXT: v_writelane_b32 v21, s5, 49
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; kill: killed $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 50
+; SI-NEXT: v_writelane_b32 v21, s5, 51
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 52
+; SI-NEXT: v_writelane_b32 v21, s5, 53
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 54
+; SI-NEXT: v_writelane_b32 v21, s5, 55
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 56
+; SI-NEXT: v_writelane_b32 v21, s5, 57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 58
+; SI-NEXT: v_writelane_b32 v21, s5, 59
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 60
+; SI-NEXT: v_writelane_b32 v21, s5, 61
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v21, s4, 62
+; SI-NEXT: v_writelane_b32 v21, s5, 63
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v23, vcc_lo, 0
-; SI-NEXT: s_bfe_u32 vcc_lo, s93, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 35
-; SI-NEXT: s_bfe_u32 vcc_lo, s92, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 37
-; SI-NEXT: s_bfe_u32 vcc_lo, s91, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 39
-; SI-NEXT: s_bfe_u32 vcc_lo, s90, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 41
-; SI-NEXT: s_bfe_u32 vcc_lo, s79, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 43
-; SI-NEXT: s_bfe_u32 vcc_lo, s78, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 45
-; SI-NEXT: s_bfe_u32 vcc_lo, s75, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 47
-; SI-NEXT: s_bfe_u32 vcc_lo, s74, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 49
-; SI-NEXT: s_bfe_u32 vcc_lo, s73, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 51
-; SI-NEXT: s_bfe_u32 vcc_lo, s72, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 53
-; SI-NEXT: s_bfe_u32 vcc_lo, s63, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 55
-; SI-NEXT: s_bfe_u32 vcc_lo, s62, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 57
-; SI-NEXT: s_bfe_u32 vcc_lo, s61, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 59
-; SI-NEXT: s_bfe_u32 vcc_lo, s60, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 61
-; SI-NEXT: s_bfe_u32 vcc_lo, s59, 0x80008
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 63
-; SI-NEXT: s_bfe_u32 vcc_lo, s58, 0x80008
-; SI-NEXT: v_writelane_b32 v23, vcc_lo, 1
-; SI-NEXT: s_lshr_b64 vcc, s[98:99], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 7
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 8
-; SI-NEXT: s_lshr_b64 vcc, s[98:99], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 5
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 6
-; SI-NEXT: s_lshr_b64 vcc, s[98:99], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 3
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 4
-; SI-NEXT: s_lshr_b64 vcc, s[82:83], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 13
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 14
-; SI-NEXT: s_lshr_b64 vcc, s[82:83], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 11
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 12
-; SI-NEXT: s_lshr_b64 vcc, s[82:83], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 9
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 10
-; SI-NEXT: s_lshr_b64 vcc, s[64:65], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 19
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 20
-; SI-NEXT: s_lshr_b64 vcc, s[64:65], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 17
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 18
-; SI-NEXT: s_lshr_b64 vcc, s[64:65], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 15
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 16
-; SI-NEXT: s_lshr_b64 vcc, s[88:89], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 25
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 26
-; SI-NEXT: s_lshr_b64 vcc, s[88:89], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 23
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 24
-; SI-NEXT: s_lshr_b64 vcc, s[88:89], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 21
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 22
-; SI-NEXT: s_lshr_b64 vcc, s[56:57], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 31
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 32
-; SI-NEXT: s_lshr_b64 vcc, s[56:57], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 29
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 30
-; SI-NEXT: s_lshr_b64 vcc, s[56:57], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 27
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 28
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 37
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 38
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 35
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 36
-; SI-NEXT: s_lshr_b64 vcc, s[76:77], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 33
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 34
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 43
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 44
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 41
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 42
-; SI-NEXT: s_lshr_b64 vcc, s[46:47], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 39
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 40
-; SI-NEXT: s_lshr_b64 vcc, s[44:45], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 49
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 50
-; SI-NEXT: s_lshr_b64 vcc, s[44:45], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 47
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 48
-; SI-NEXT: s_lshr_b64 vcc, s[44:45], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 45
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 46
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 55
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 56
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 53
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 54
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 51
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 52
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 24
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 61
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 62
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 16
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 59
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 60
-; SI-NEXT: s_lshr_b64 vcc, s[40:41], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 57
-; SI-NEXT: v_writelane_b32 v21, vcc_hi, 58
-; SI-NEXT: s_lshr_b64 vcc, s[14:15], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 3
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 4
-; SI-NEXT: s_lshr_b64 vcc, s[14:15], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 1
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 2
-; SI-NEXT: s_lshr_b64 vcc, s[14:15], 8
-; SI-NEXT: v_writelane_b32 v21, vcc_lo, 63
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 0
-; SI-NEXT: s_lshr_b64 vcc, s[12:13], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 9
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 10
-; SI-NEXT: s_lshr_b64 vcc, s[12:13], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 7
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 8
-; SI-NEXT: s_lshr_b64 vcc, s[12:13], 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 5
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 6
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 15
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 16
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 13
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 14
-; SI-NEXT: s_lshr_b64 vcc, s[10:11], 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 11
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 12
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 21
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 22
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 19
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 20
-; SI-NEXT: s_lshr_b64 vcc, s[8:9], 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 17
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 18
-; SI-NEXT: s_lshr_b64 vcc, s[6:7], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 27
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 28
-; SI-NEXT: s_lshr_b64 vcc, s[6:7], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 25
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 26
-; SI-NEXT: s_lshr_b64 vcc, s[6:7], 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 23
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 24
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 33
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 34
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 31
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 32
-; SI-NEXT: s_lshr_b64 vcc, s[4:5], 8
-; SI-NEXT: v_writelane_b32 v22, vcc_lo, 29
-; SI-NEXT: s_lshr_b32 s52, s5, 8
-; SI-NEXT: v_writelane_b32 v22, vcc_hi, 30
-; SI-NEXT: s_cbranch_execnz .LBB99_3
-; SI-NEXT: .LBB99_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s51, s51, 3
-; SI-NEXT: s_and_b32 s4, s51, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
+; SI-NEXT: v_writelane_b32 v22, s4, 0
+; SI-NEXT: v_writelane_b32 v22, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 2
+; SI-NEXT: v_writelane_b32 v22, s5, 3
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 4
+; SI-NEXT: v_writelane_b32 v22, s5, 5
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 6
+; SI-NEXT: v_writelane_b32 v22, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 8
+; SI-NEXT: v_writelane_b32 v22, s5, 9
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 10
+; SI-NEXT: v_writelane_b32 v22, s5, 11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 12
+; SI-NEXT: v_writelane_b32 v22, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 14
+; SI-NEXT: v_writelane_b32 v22, s5, 15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 16
+; SI-NEXT: v_writelane_b32 v22, s5, 17
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 18
+; SI-NEXT: v_writelane_b32 v22, s5, 19
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 20
+; SI-NEXT: v_writelane_b32 v22, s5, 21
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 22
+; SI-NEXT: v_writelane_b32 v22, s5, 23
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 24
+; SI-NEXT: v_writelane_b32 v22, s5, 25
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 26
+; SI-NEXT: v_writelane_b32 v22, s5, 27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 28
+; SI-NEXT: v_writelane_b32 v22, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 30
+; SI-NEXT: v_writelane_b32 v22, s5, 31
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 32
+; SI-NEXT: v_writelane_b32 v22, s5, 33
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 34
+; SI-NEXT: v_writelane_b32 v22, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 36
+; SI-NEXT: v_writelane_b32 v22, s5, 37
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 38
+; SI-NEXT: v_writelane_b32 v22, s5, 39
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 40
+; SI-NEXT: v_writelane_b32 v22, s5, 41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 42
+; SI-NEXT: v_writelane_b32 v22, s5, 43
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 44
+; SI-NEXT: v_writelane_b32 v22, s5, 45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 46
+; SI-NEXT: v_writelane_b32 v22, s5, 47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v22, s4, 48
+; SI-NEXT: v_writelane_b32 v22, s5, 49
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB99_3: ; %Flow
+; SI-NEXT: s_and_b64 s[96:97], s[96:97], exec
+; SI-NEXT: s_cselect_b32 s96, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s96, 1
+; SI-NEXT: s_mov_b32 s96, s50
+; SI-NEXT: s_mov_b32 s50, s52
+; SI-NEXT: s_mov_b32 s52, s60
+; SI-NEXT: s_mov_b32 s60, s98
+; SI-NEXT: s_mov_b32 s97, s94
+; SI-NEXT: s_mov_b32 s94, s67
+; SI-NEXT: s_mov_b32 s67, s70
+; SI-NEXT: s_mov_b32 s70, s31
+; SI-NEXT: s_mov_b32 s31, s87
+; SI-NEXT: s_mov_b32 s87, s92
+; SI-NEXT: s_mov_b32 s92, s48
+; SI-NEXT: s_mov_b32 s48, s74
+; SI-NEXT: s_mov_b32 s74, s78
+; SI-NEXT: s_mov_b32 s78, s90
+; SI-NEXT: v_readlane_b32 s90, v22, 50
+; SI-NEXT: v_readlane_b32 s98, v22, 51
+; SI-NEXT: s_cbranch_scc1 .LBB99_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s49, s49, 3
+; SI-NEXT: v_readlane_b32 s5, v21, 25
+; SI-NEXT: s_and_b32 s4, s49, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_add_i32 s39, s39, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s49, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s58, 16
-; SI-NEXT: s_add_i32 s48, s48, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s48, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s39, 16
+; SI-NEXT: s_and_b32 s5, s39, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s59, 16
; SI-NEXT: s_add_i32 s38, s38, 3
+; SI-NEXT: v_readlane_b32 s7, v21, 24
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s38, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_add_i32 s36, s36, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s38, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s59, 16
-; SI-NEXT: s_add_i32 s95, s95, 3
+; SI-NEXT: s_and_b32 s7, s36, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s51, 16
+; SI-NEXT: s_add_i32 s93, s93, 3
+; SI-NEXT: v_readlane_b32 s9, v21, 23
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s94, 16
-; SI-NEXT: s_add_i32 s37, s37, 3
+; SI-NEXT: s_and_b32 s8, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s37, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s60, 16
-; SI-NEXT: s_add_i32 s36, s36, 3
+; SI-NEXT: s_and_b32 s9, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s61, 16
+; SI-NEXT: s_add_i32 s34, s34, 3
+; SI-NEXT: v_readlane_b32 s11, v21, 22
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s36, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s97, 16
-; SI-NEXT: s_add_i32 s96, s96, 3
+; SI-NEXT: s_and_b32 s10, s34, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_add_i32 s86, s86, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s96, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s61, 16
-; SI-NEXT: s_add_i32 s87, s87, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s87, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s86, 16
+; SI-NEXT: s_and_b32 s11, s86, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s53, 16
; SI-NEXT: s_add_i32 s85, s85, 3
+; SI-NEXT: v_readlane_b32 s13, v21, 21
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s12, s85, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s13, 16
+; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s62, 16
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_and_b32 s13, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s97, 16
+; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: v_readlane_b32 s15, v21, 20
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s35, 16
-; SI-NEXT: s_add_i32 s34, s34, 3
+; SI-NEXT: s_and_b32 s14, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: v_readlane_b32 s40, v21, 0
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_and_b32 s15, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s63, 16
-; SI-NEXT: s_add_i32 s81, s81, 3
-; SI-NEXT: s_or_b32 s15, s40, s15
-; SI-NEXT: s_and_b32 s40, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s80, 16
+; SI-NEXT: s_and_b32 s15, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s40, s40, 16
; SI-NEXT: s_add_i32 s71, s71, 3
+; SI-NEXT: v_readlane_b32 s41, v21, 19
+; SI-NEXT: s_or_b32 s15, s40, s15
+; SI-NEXT: s_and_b32 s40, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s41, s41, 16
+; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: v_readlane_b32 s42, v21, 1
; SI-NEXT: s_or_b32 s40, s41, s40
-; SI-NEXT: s_and_b32 s41, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s72, 16
-; SI-NEXT: s_add_i32 s70, s70, 3
-; SI-NEXT: s_or_b32 s41, s42, s41
-; SI-NEXT: s_and_b32 s42, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s69, 16
+; SI-NEXT: s_and_b32 s41, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: v_readlane_b32 s43, v21, 18
+; SI-NEXT: s_or_b32 s41, s42, s41
+; SI-NEXT: s_and_b32 s42, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s43, s43, 16
+; SI-NEXT: s_add_i32 s66, s66, 3
+; SI-NEXT: v_readlane_b32 s44, v21, 2
; SI-NEXT: s_or_b32 s42, s43, s42
-; SI-NEXT: s_and_b32 s43, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s73, 16
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_and_b32 s43, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s44, s44, 16
+; SI-NEXT: s_add_i32 s95, s95, 3
+; SI-NEXT: v_readlane_b32 s45, v21, 17
; SI-NEXT: s_or_b32 s43, s44, s43
-; SI-NEXT: s_and_b32 s44, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s45, s30, 16
-; SI-NEXT: s_add_i32 s67, s67, 3
+; SI-NEXT: s_and_b32 s44, s95, 0xffff
+; SI-NEXT: s_lshl_b32 s45, s45, 16
+; SI-NEXT: s_add_i32 s55, s55, 3
+; SI-NEXT: v_readlane_b32 s46, v21, 3
; SI-NEXT: s_or_b32 s44, s45, s44
-; SI-NEXT: s_and_b32 s45, s67, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s74, 16
-; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_and_b32 s45, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s46, 16
; SI-NEXT: s_or_b32 s45, s46, s45
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: v_readlane_b32 s46, v21, 16
; SI-NEXT: s_and_b32 s28, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s66, 16
+; SI-NEXT: s_lshl_b32 s46, s46, 16
; SI-NEXT: s_or_b32 s28, s46, s28
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s46, s28, 0x30000
; SI-NEXT: s_and_b32 s28, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s75, 16
+; SI-NEXT: v_readlane_b32 s29, v21, 4
+; SI-NEXT: s_lshl_b32 s29, s29, 16
; SI-NEXT: s_or_b32 s28, s29, s28
-; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s47, s28, 0x30000
+; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: v_readlane_b32 s28, v21, 15
; SI-NEXT: s_and_b32 s26, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s28, s55, 16
+; SI-NEXT: s_lshl_b32 s28, s28, 16
; SI-NEXT: s_or_b32 s26, s28, s26
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s76, s26, 0x30000
; SI-NEXT: s_and_b32 s26, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s78, 16
+; SI-NEXT: v_readlane_b32 s27, v21, 5
+; SI-NEXT: s_lshl_b32 s27, s27, 16
; SI-NEXT: s_or_b32 s26, s27, s26
-; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s77, s26, 0x30000
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: v_readlane_b32 s26, v21, 14
; SI-NEXT: s_and_b32 s24, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s54, 16
+; SI-NEXT: s_lshl_b32 s26, s26, 16
; SI-NEXT: s_or_b32 s24, s26, s24
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s56, s24, 0x30000
; SI-NEXT: s_and_b32 s24, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s79, 16
+; SI-NEXT: v_readlane_b32 s25, v21, 6
+; SI-NEXT: s_lshl_b32 s25, s25, 16
; SI-NEXT: s_or_b32 s24, s25, s24
; SI-NEXT: s_add_i32 s57, s24, 0x30000
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: v_readlane_b32 s24, v21, 2
+; SI-NEXT: v_readlane_b32 s24, v21, 13
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_lshl_b32 s24, s24, 16
; SI-NEXT: s_or_b32 s22, s24, s22
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s88, s22, 0x30000
; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: v_readlane_b32 s23, v21, 7
+; SI-NEXT: s_lshl_b32 s23, s23, 16
; SI-NEXT: s_or_b32 s22, s23, s22
-; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s89, s22, 0x30000
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: v_readlane_b32 s22, v21, 12
; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s53, 16
+; SI-NEXT: s_lshl_b32 s22, s22, 16
; SI-NEXT: s_or_b32 s20, s22, s20
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s64, s20, 0x30000
; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s91, 16
+; SI-NEXT: v_readlane_b32 s21, v21, 8
+; SI-NEXT: s_lshl_b32 s21, s21, 16
; SI-NEXT: s_or_b32 s20, s21, s20
; SI-NEXT: s_add_i32 s65, s20, 0x30000
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: v_readlane_b32 s20, v21, 1
+; SI-NEXT: v_readlane_b32 s20, v21, 11
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s82, s18, 0x30000
; SI-NEXT: s_and_b32 s18, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s92, 16
+; SI-NEXT: v_readlane_b32 s19, v21, 9
+; SI-NEXT: s_lshl_b32 s19, s19, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_add_i32 s83, s18, 0x30000
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s18, v21, 0
+; SI-NEXT: v_readlane_b32 s18, v21, 10
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s98, s16, 0x30000
+; SI-NEXT: s_add_i32 vcc_lo, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s93, 16
+; SI-NEXT: s_lshl_b32 s17, s91, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s99, s16, 0x30000
-; SI-NEXT: s_lshr_b64 s[16:17], s[98:99], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 7
-; SI-NEXT: v_writelane_b32 v21, s17, 8
-; SI-NEXT: s_lshr_b64 s[16:17], s[98:99], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 5
-; SI-NEXT: v_writelane_b32 v21, s17, 6
-; SI-NEXT: s_lshr_b64 s[16:17], s[98:99], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 3
-; SI-NEXT: v_writelane_b32 v21, s17, 4
+; SI-NEXT: s_add_i32 vcc_hi, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[16:17], vcc, 24
+; SI-NEXT: v_writelane_b32 v21, s16, 30
+; SI-NEXT: v_writelane_b32 v21, s17, 31
+; SI-NEXT: s_lshr_b64 s[16:17], vcc, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 28
+; SI-NEXT: v_writelane_b32 v21, s17, 29
+; SI-NEXT: s_lshr_b64 s[16:17], vcc, 8
+; SI-NEXT: v_writelane_b32 v21, s16, 26
+; SI-NEXT: v_writelane_b32 v21, s17, 27
; SI-NEXT: s_lshr_b64 s[16:17], s[82:83], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 13
-; SI-NEXT: v_writelane_b32 v21, s17, 14
+; SI-NEXT: v_writelane_b32 v21, s16, 36
+; SI-NEXT: v_writelane_b32 v21, s17, 37
; SI-NEXT: s_lshr_b64 s[16:17], s[82:83], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 11
-; SI-NEXT: v_writelane_b32 v21, s17, 12
+; SI-NEXT: v_writelane_b32 v21, s16, 34
+; SI-NEXT: v_writelane_b32 v21, s17, 35
; SI-NEXT: s_lshr_b64 s[16:17], s[82:83], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 9
-; SI-NEXT: v_writelane_b32 v21, s17, 10
+; SI-NEXT: v_writelane_b32 v21, s16, 32
+; SI-NEXT: v_writelane_b32 v21, s17, 33
; SI-NEXT: s_lshr_b64 s[16:17], s[64:65], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 19
-; SI-NEXT: v_writelane_b32 v21, s17, 20
+; SI-NEXT: v_writelane_b32 v21, s16, 42
+; SI-NEXT: v_writelane_b32 v21, s17, 43
; SI-NEXT: s_lshr_b64 s[16:17], s[64:65], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 17
-; SI-NEXT: v_writelane_b32 v21, s17, 18
+; SI-NEXT: v_writelane_b32 v21, s16, 40
+; SI-NEXT: v_writelane_b32 v21, s17, 41
; SI-NEXT: s_lshr_b64 s[16:17], s[64:65], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 15
-; SI-NEXT: v_writelane_b32 v21, s17, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 38
+; SI-NEXT: v_writelane_b32 v21, s17, 39
; SI-NEXT: s_lshr_b64 s[16:17], s[88:89], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 25
-; SI-NEXT: v_writelane_b32 v21, s17, 26
+; SI-NEXT: v_writelane_b32 v21, s16, 48
+; SI-NEXT: v_writelane_b32 v21, s17, 49
; SI-NEXT: s_lshr_b64 s[16:17], s[88:89], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 23
-; SI-NEXT: v_writelane_b32 v21, s17, 24
+; SI-NEXT: v_writelane_b32 v21, s16, 46
+; SI-NEXT: v_writelane_b32 v21, s17, 47
; SI-NEXT: s_lshr_b64 s[16:17], s[88:89], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 21
-; SI-NEXT: v_writelane_b32 v21, s17, 22
+; SI-NEXT: v_writelane_b32 v21, s16, 44
+; SI-NEXT: v_writelane_b32 v21, s17, 45
; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 31
-; SI-NEXT: v_writelane_b32 v21, s17, 32
+; SI-NEXT: v_writelane_b32 v21, s16, 54
+; SI-NEXT: v_writelane_b32 v21, s17, 55
; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 29
-; SI-NEXT: v_writelane_b32 v21, s17, 30
+; SI-NEXT: v_writelane_b32 v21, s16, 52
+; SI-NEXT: v_writelane_b32 v21, s17, 53
; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 27
-; SI-NEXT: v_writelane_b32 v21, s17, 28
+; SI-NEXT: v_writelane_b32 v21, s16, 50
+; SI-NEXT: v_writelane_b32 v21, s17, 51
; SI-NEXT: s_lshr_b64 s[16:17], s[76:77], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 37
-; SI-NEXT: v_writelane_b32 v21, s17, 38
+; SI-NEXT: v_writelane_b32 v21, s16, 60
+; SI-NEXT: v_writelane_b32 v21, s17, 61
; SI-NEXT: s_lshr_b64 s[16:17], s[76:77], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 35
-; SI-NEXT: v_writelane_b32 v21, s17, 36
+; SI-NEXT: v_writelane_b32 v21, s16, 58
+; SI-NEXT: v_writelane_b32 v21, s17, 59
; SI-NEXT: s_lshr_b64 s[16:17], s[76:77], 8
-; SI-NEXT: v_writelane_b32 v21, s16, 33
-; SI-NEXT: v_writelane_b32 v21, s17, 34
+; SI-NEXT: v_writelane_b32 v21, s16, 56
+; SI-NEXT: v_writelane_b32 v21, s17, 57
; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 43
-; SI-NEXT: v_writelane_b32 v21, s17, 44
+; SI-NEXT: v_writelane_b32 v22, s16, 2
+; SI-NEXT: v_writelane_b32 v22, s17, 3
; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 41
-; SI-NEXT: v_writelane_b32 v21, s17, 42
+; SI-NEXT: v_writelane_b32 v22, s16, 0
+; SI-NEXT: v_writelane_b32 v22, s17, 1
; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 8
; SI-NEXT: s_add_i32 s44, s44, 0x30000
; SI-NEXT: s_add_i32 s45, s45, 0x30000
-; SI-NEXT: v_writelane_b32 v21, s16, 39
-; SI-NEXT: v_writelane_b32 v21, s17, 40
+; SI-NEXT: v_writelane_b32 v21, s16, 62
+; SI-NEXT: v_writelane_b32 v21, s17, 63
; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 49
-; SI-NEXT: v_writelane_b32 v21, s17, 50
+; SI-NEXT: v_writelane_b32 v22, s16, 8
+; SI-NEXT: v_writelane_b32 v22, s17, 9
; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 47
-; SI-NEXT: v_writelane_b32 v21, s17, 48
+; SI-NEXT: v_writelane_b32 v22, s16, 6
+; SI-NEXT: v_writelane_b32 v22, s17, 7
; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 8
; SI-NEXT: s_add_i32 s42, s42, 0x30000
; SI-NEXT: s_add_i32 s43, s43, 0x30000
-; SI-NEXT: v_writelane_b32 v21, s16, 45
-; SI-NEXT: v_writelane_b32 v21, s17, 46
+; SI-NEXT: v_writelane_b32 v22, s16, 4
+; SI-NEXT: v_writelane_b32 v22, s17, 5
; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 55
-; SI-NEXT: v_writelane_b32 v21, s17, 56
+; SI-NEXT: v_writelane_b32 v22, s16, 14
+; SI-NEXT: v_writelane_b32 v22, s17, 15
; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 53
-; SI-NEXT: v_writelane_b32 v21, s17, 54
+; SI-NEXT: v_writelane_b32 v22, s16, 12
+; SI-NEXT: v_writelane_b32 v22, s17, 13
; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 8
; SI-NEXT: s_add_i32 s40, s40, 0x30000
; SI-NEXT: s_add_i32 s41, s41, 0x30000
-; SI-NEXT: v_writelane_b32 v21, s16, 51
-; SI-NEXT: v_writelane_b32 v21, s17, 52
+; SI-NEXT: v_writelane_b32 v22, s16, 10
+; SI-NEXT: v_writelane_b32 v22, s17, 11
; SI-NEXT: s_lshr_b64 s[16:17], s[40:41], 24
-; SI-NEXT: v_writelane_b32 v21, s16, 61
-; SI-NEXT: v_writelane_b32 v21, s17, 62
+; SI-NEXT: v_writelane_b32 v22, s16, 20
+; SI-NEXT: v_writelane_b32 v22, s17, 21
; SI-NEXT: s_lshr_b64 s[16:17], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v21, s16, 59
-; SI-NEXT: v_writelane_b32 v21, s17, 60
+; SI-NEXT: v_writelane_b32 v22, s16, 18
+; SI-NEXT: v_writelane_b32 v22, s17, 19
; SI-NEXT: s_lshr_b64 s[16:17], s[40:41], 8
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: v_writelane_b32 v21, s16, 57
-; SI-NEXT: v_writelane_b32 v21, s17, 58
+; SI-NEXT: v_writelane_b32 v22, s16, 16
+; SI-NEXT: v_writelane_b32 v22, s17, 17
; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 3
-; SI-NEXT: v_writelane_b32 v22, s17, 4
+; SI-NEXT: v_writelane_b32 v22, s16, 26
+; SI-NEXT: v_writelane_b32 v22, s17, 27
; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 1
-; SI-NEXT: v_writelane_b32 v22, s17, 2
+; SI-NEXT: v_writelane_b32 v22, s16, 24
+; SI-NEXT: v_writelane_b32 v22, s17, 25
; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 8
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: v_writelane_b32 v21, s16, 63
-; SI-NEXT: v_writelane_b32 v22, s17, 0
+; SI-NEXT: v_writelane_b32 v22, s16, 22
+; SI-NEXT: v_writelane_b32 v22, s17, 23
; SI-NEXT: s_lshr_b64 s[16:17], s[12:13], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 9
-; SI-NEXT: v_writelane_b32 v22, s17, 10
+; SI-NEXT: v_writelane_b32 v22, s16, 32
+; SI-NEXT: v_writelane_b32 v22, s17, 33
; SI-NEXT: s_lshr_b64 s[16:17], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 7
-; SI-NEXT: v_writelane_b32 v22, s17, 8
+; SI-NEXT: v_writelane_b32 v22, s16, 30
+; SI-NEXT: v_writelane_b32 v22, s17, 31
; SI-NEXT: s_lshr_b64 s[16:17], s[12:13], 8
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: v_writelane_b32 v22, s16, 5
-; SI-NEXT: v_writelane_b32 v22, s17, 6
+; SI-NEXT: v_writelane_b32 v22, s16, 28
+; SI-NEXT: v_writelane_b32 v22, s17, 29
; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 15
-; SI-NEXT: v_writelane_b32 v22, s17, 16
+; SI-NEXT: v_writelane_b32 v22, s16, 38
+; SI-NEXT: v_writelane_b32 v22, s17, 39
; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 13
-; SI-NEXT: v_writelane_b32 v22, s17, 14
+; SI-NEXT: v_writelane_b32 v22, s16, 36
+; SI-NEXT: v_writelane_b32 v22, s17, 37
; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 8
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: v_writelane_b32 v22, s16, 11
-; SI-NEXT: v_writelane_b32 v22, s17, 12
+; SI-NEXT: v_writelane_b32 v22, s16, 34
+; SI-NEXT: v_writelane_b32 v22, s17, 35
; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 21
-; SI-NEXT: v_writelane_b32 v22, s17, 22
+; SI-NEXT: v_writelane_b32 v22, s16, 44
+; SI-NEXT: v_writelane_b32 v22, s17, 45
; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 19
-; SI-NEXT: v_writelane_b32 v22, s17, 20
+; SI-NEXT: v_writelane_b32 v22, s16, 42
+; SI-NEXT: v_writelane_b32 v22, s17, 43
; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 8
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: v_writelane_b32 v22, s16, 17
-; SI-NEXT: v_writelane_b32 v22, s17, 18
-; SI-NEXT: s_lshr_b64 s[16:17], s[6:7], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 27
-; SI-NEXT: v_writelane_b32 v22, s17, 28
+; SI-NEXT: v_writelane_b32 v22, s16, 40
+; SI-NEXT: v_writelane_b32 v22, s17, 41
; SI-NEXT: s_lshr_b64 s[16:17], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 25
-; SI-NEXT: v_writelane_b32 v22, s17, 26
+; SI-NEXT: v_writelane_b32 v22, s16, 48
+; SI-NEXT: v_writelane_b32 v22, s17, 49
; SI-NEXT: s_lshr_b64 s[16:17], s[6:7], 8
-; SI-NEXT: s_add_i32 s4, s4, 0x30000
-; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: v_writelane_b32 v22, s16, 23
-; SI-NEXT: v_writelane_b32 v22, s17, 24
-; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 24
-; SI-NEXT: v_writelane_b32 v22, s16, 33
-; SI-NEXT: v_writelane_b32 v22, s17, 34
-; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v22, s16, 31
-; SI-NEXT: v_writelane_b32 v22, s17, 32
-; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 8
-; SI-NEXT: v_writelane_b32 v22, s16, 29
-; SI-NEXT: v_writelane_b32 v22, s17, 30
-; SI-NEXT: s_lshr_b32 s16, s99, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 35
-; SI-NEXT: s_lshr_b32 s16, s99, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 36
-; SI-NEXT: s_lshr_b32 s16, s83, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 37
-; SI-NEXT: s_lshr_b32 s16, s83, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 38
-; SI-NEXT: s_lshr_b32 s16, s65, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 39
-; SI-NEXT: s_lshr_b32 s16, s65, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 40
-; SI-NEXT: s_lshr_b32 s16, s89, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 41
-; SI-NEXT: s_lshr_b32 s16, s89, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 42
-; SI-NEXT: s_lshr_b32 s16, s57, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 43
-; SI-NEXT: s_lshr_b32 s16, s57, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 44
-; SI-NEXT: s_lshr_b32 s16, s77, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 45
-; SI-NEXT: s_lshr_b32 s16, s77, 8
; SI-NEXT: v_writelane_b32 v22, s16, 46
-; SI-NEXT: s_lshr_b32 s16, s47, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 47
-; SI-NEXT: s_lshr_b32 s16, s47, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 48
-; SI-NEXT: s_lshr_b32 s16, s45, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 49
-; SI-NEXT: s_lshr_b32 s16, s45, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 50
-; SI-NEXT: s_lshr_b32 s16, s43, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 51
-; SI-NEXT: s_lshr_b32 s16, s43, 8
+; SI-NEXT: v_writelane_b32 v22, s17, 47
+; SI-NEXT: s_lshr_b32 s16, vcc_hi, 24
; SI-NEXT: v_writelane_b32 v22, s16, 52
-; SI-NEXT: s_lshr_b32 s16, s41, 24
+; SI-NEXT: s_lshr_b32 s16, vcc_hi, 8
; SI-NEXT: v_writelane_b32 v22, s16, 53
-; SI-NEXT: s_lshr_b32 s16, s41, 8
+; SI-NEXT: s_lshr_b32 s16, s83, 24
; SI-NEXT: v_writelane_b32 v22, s16, 54
-; SI-NEXT: s_lshr_b32 s16, s15, 24
+; SI-NEXT: s_lshr_b32 s16, s83, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 9
+; SI-NEXT: s_lshr_b32 s16, s83, 8
; SI-NEXT: v_writelane_b32 v22, s16, 55
-; SI-NEXT: s_lshr_b32 s16, s15, 8
+; SI-NEXT: s_lshr_b32 s16, s65, 24
; SI-NEXT: v_writelane_b32 v22, s16, 56
-; SI-NEXT: s_lshr_b32 s16, s13, 24
+; SI-NEXT: s_lshr_b32 s16, s65, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 8
+; SI-NEXT: s_lshr_b32 s16, s65, 8
; SI-NEXT: v_writelane_b32 v22, s16, 57
-; SI-NEXT: s_lshr_b32 s16, s13, 8
+; SI-NEXT: s_lshr_b32 s16, s89, 24
; SI-NEXT: v_writelane_b32 v22, s16, 58
-; SI-NEXT: s_lshr_b32 s16, s11, 24
+; SI-NEXT: s_lshr_b32 s16, s89, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 7
+; SI-NEXT: s_lshr_b32 s16, s89, 8
; SI-NEXT: v_writelane_b32 v22, s16, 59
-; SI-NEXT: s_lshr_b32 s16, s11, 8
+; SI-NEXT: s_lshr_b32 s16, s57, 24
; SI-NEXT: v_writelane_b32 v22, s16, 60
-; SI-NEXT: s_lshr_b32 s16, s9, 24
+; SI-NEXT: s_lshr_b32 s16, s57, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 6
+; SI-NEXT: s_lshr_b32 s16, s57, 8
; SI-NEXT: v_writelane_b32 v22, s16, 61
-; SI-NEXT: s_lshr_b32 s16, s9, 8
-; SI-NEXT: v_writelane_b32 v22, s16, 62
-; SI-NEXT: s_lshr_b32 s16, s7, 24
-; SI-NEXT: v_writelane_b32 v22, s16, 63
-; SI-NEXT: s_lshr_b32 s16, s7, 8
-; SI-NEXT: s_lshr_b32 s93, s99, 16
-; SI-NEXT: s_lshr_b32 s92, s83, 16
-; SI-NEXT: s_lshr_b32 s91, s65, 16
-; SI-NEXT: s_lshr_b32 s90, s89, 16
-; SI-NEXT: s_lshr_b32 s79, s57, 16
-; SI-NEXT: s_lshr_b32 s78, s77, 16
-; SI-NEXT: s_lshr_b32 s75, s47, 16
-; SI-NEXT: s_lshr_b32 s74, s45, 16
-; SI-NEXT: s_lshr_b32 s73, s43, 16
-; SI-NEXT: s_lshr_b32 s72, s41, 16
-; SI-NEXT: s_lshr_b32 s63, s15, 16
-; SI-NEXT: s_lshr_b32 s62, s13, 16
-; SI-NEXT: s_lshr_b32 s61, s11, 16
-; SI-NEXT: s_lshr_b32 s60, s9, 16
-; SI-NEXT: s_lshr_b32 s59, s7, 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v23, s16, 0
-; SI-NEXT: s_lshr_b32 s16, s5, 24
-; SI-NEXT: s_lshr_b32 s58, s5, 16
-; SI-NEXT: s_lshr_b32 s52, s5, 8
-; SI-NEXT: v_writelane_b32 v23, s16, 1
-; SI-NEXT: .LBB99_3: ; %end
-; SI-NEXT: v_readlane_b32 s18, v21, 3
-; SI-NEXT: v_readlane_b32 s19, v21, 4
+; SI-NEXT: s_lshr_b32 s16, s77, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 5
+; SI-NEXT: s_lshr_b32 s16, s47, 16
+; SI-NEXT: v_writelane_b32 v21, s16, 4
+; SI-NEXT: s_lshr_b32 s16, s45, 16
+; SI-NEXT: s_add_i32 s4, s4, 0x30000
+; SI-NEXT: s_add_i32 s5, s5, 0x30000
+; SI-NEXT: v_writelane_b32 v21, s16, 3
+; SI-NEXT: s_lshr_b32 s16, s43, 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[6:7], 24
+; SI-NEXT: s_lshr_b64 s[96:97], s[4:5], 24
+; SI-NEXT: s_lshr_b64 s[50:51], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[4:5], 8
+; SI-NEXT: v_writelane_b32 v21, s16, 2
+; SI-NEXT: s_lshr_b32 s16, s41, 16
+; SI-NEXT: s_lshr_b32 s91, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s98, s77, 24
+; SI-NEXT: s_lshr_b32 s90, s77, 8
+; SI-NEXT: s_lshr_b32 s79, s47, 24
+; SI-NEXT: s_lshr_b32 s78, s47, 8
+; SI-NEXT: s_lshr_b32 s75, s45, 24
+; SI-NEXT: s_lshr_b32 s74, s45, 8
+; SI-NEXT: s_lshr_b32 s73, s43, 24
+; SI-NEXT: s_lshr_b32 s48, s43, 8
+; SI-NEXT: s_lshr_b32 s37, s41, 24
+; SI-NEXT: v_writelane_b32 v21, s16, 1
+; SI-NEXT: s_lshr_b32 s92, s41, 8
+; SI-NEXT: s_lshr_b32 s72, s15, 24
+; SI-NEXT: s_lshr_b32 s16, s15, 16
+; SI-NEXT: s_lshr_b32 s87, s15, 8
+; SI-NEXT: s_lshr_b32 s84, s13, 24
+; SI-NEXT: s_lshr_b32 s97, s13, 16
+; SI-NEXT: s_lshr_b32 s31, s13, 8
+; SI-NEXT: s_lshr_b32 s63, s11, 24
+; SI-NEXT: s_lshr_b32 s53, s11, 16
+; SI-NEXT: s_lshr_b32 s70, s11, 8
+; SI-NEXT: s_lshr_b32 s62, s9, 24
+; SI-NEXT: s_lshr_b32 s61, s9, 16
+; SI-NEXT: s_lshr_b32 s67, s9, 8
+; SI-NEXT: s_lshr_b32 s54, s7, 24
+; SI-NEXT: s_lshr_b32 s51, s7, 16
+; SI-NEXT: s_lshr_b32 s94, s7, 8
+; SI-NEXT: s_lshr_b32 s60, s5, 24
+; SI-NEXT: s_lshr_b32 s59, s5, 16
+; SI-NEXT: s_lshr_b32 s99, s5, 8
+; SI-NEXT: v_writelane_b32 v21, s16, 0
+; SI-NEXT: .LBB99_5: ; %end
+; SI-NEXT: v_readlane_b32 s18, v21, 26
+; SI-NEXT: v_readlane_b32 s19, v21, 27
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 5
-; SI-NEXT: s_and_b32 s16, s98, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 6
+; SI-NEXT: v_readlane_b32 s18, v21, 28
+; SI-NEXT: s_and_b32 s16, vcc_lo, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 29
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 7
+; SI-NEXT: v_readlane_b32 s18, v21, 30
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 36
+; SI-NEXT: v_readlane_b32 s17, v22, 53
; SI-NEXT: v_mov_b32_e32 v1, s16
-; SI-NEXT: s_and_b32 s16, s99, 0xff
+; SI-NEXT: s_and_b32 s16, vcc_hi, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s93, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 35
-; SI-NEXT: v_readlane_b32 s19, v21, 8
+; SI-NEXT: s_and_b32 s17, s91, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 52
+; SI-NEXT: v_readlane_b32 s19, v21, 31
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 9
+; SI-NEXT: v_readlane_b32 s18, v21, 32
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 10
+; SI-NEXT: v_readlane_b32 s19, v21, 33
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 11
+; SI-NEXT: v_readlane_b32 s18, v21, 34
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s82, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 12
+; SI-NEXT: v_readlane_b32 s19, v21, 35
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 13
+; SI-NEXT: v_readlane_b32 s18, v21, 36
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -203067,289 +204105,288 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 38
+; SI-NEXT: v_readlane_b32 s17, v22, 55
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s83, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s92, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 37
-; SI-NEXT: v_readlane_b32 s19, v21, 14
+; SI-NEXT: v_readlane_b32 s17, v21, 9
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 54
+; SI-NEXT: v_readlane_b32 s19, v21, 37
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 15
+; SI-NEXT: v_readlane_b32 s18, v21, 38
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 16
+; SI-NEXT: v_readlane_b32 s19, v21, 39
; SI-NEXT: v_add_i32_e32 v1, vcc, 8, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 17
+; SI-NEXT: v_readlane_b32 s18, v21, 40
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s64, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 18
+; SI-NEXT: v_readlane_b32 s19, v21, 41
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 19
+; SI-NEXT: v_readlane_b32 s18, v21, 42
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 12, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 40
+; SI-NEXT: v_readlane_b32 s17, v22, 57
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s65, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s91, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 39
-; SI-NEXT: v_readlane_b32 s19, v21, 20
+; SI-NEXT: v_readlane_b32 s17, v21, 8
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 56
+; SI-NEXT: v_readlane_b32 s19, v21, 43
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 21
+; SI-NEXT: v_readlane_b32 s18, v21, 44
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 22
+; SI-NEXT: v_readlane_b32 s19, v21, 45
; SI-NEXT: v_add_i32_e32 v1, vcc, 16, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 23
+; SI-NEXT: v_readlane_b32 s18, v21, 46
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s88, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 24
+; SI-NEXT: v_readlane_b32 s19, v21, 47
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 25
+; SI-NEXT: v_readlane_b32 s18, v21, 48
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 20, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 42
+; SI-NEXT: v_readlane_b32 s17, v22, 59
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s89, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s90, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 41
-; SI-NEXT: v_readlane_b32 s19, v21, 26
+; SI-NEXT: v_readlane_b32 s17, v21, 7
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 58
+; SI-NEXT: v_readlane_b32 s19, v21, 49
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 27
+; SI-NEXT: v_readlane_b32 s18, v21, 50
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 28
+; SI-NEXT: v_readlane_b32 s19, v21, 51
; SI-NEXT: v_add_i32_e32 v1, vcc, 24, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 29
+; SI-NEXT: v_readlane_b32 s18, v21, 52
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s56, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 30
+; SI-NEXT: v_readlane_b32 s19, v21, 53
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 31
+; SI-NEXT: v_readlane_b32 s18, v21, 54
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 28, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 44
+; SI-NEXT: v_readlane_b32 s17, v22, 61
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s57, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s79, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 43
-; SI-NEXT: v_readlane_b32 s19, v21, 32
+; SI-NEXT: v_readlane_b32 s17, v21, 6
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v22, 60
+; SI-NEXT: v_readlane_b32 s19, v21, 55
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 33
+; SI-NEXT: v_readlane_b32 s18, v21, 56
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 34
+; SI-NEXT: v_readlane_b32 s19, v21, 57
; SI-NEXT: v_add_i32_e32 v1, vcc, 32, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 35
+; SI-NEXT: v_readlane_b32 s18, v21, 58
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s76, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 36
+; SI-NEXT: v_readlane_b32 s19, v21, 59
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 37
+; SI-NEXT: v_readlane_b32 s18, v21, 60
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 36, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 46
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s77, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s90, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s78, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 45
-; SI-NEXT: v_readlane_b32 s19, v21, 38
+; SI-NEXT: v_readlane_b32 s17, v21, 5
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s19, v21, 61
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s98, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 39
+; SI-NEXT: v_readlane_b32 s18, v21, 62
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 40
+; SI-NEXT: v_readlane_b32 s19, v21, 63
; SI-NEXT: v_add_i32_e32 v1, vcc, 40, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 41
+; SI-NEXT: v_readlane_b32 s18, v22, 0
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s46, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 42
+; SI-NEXT: v_readlane_b32 s19, v22, 1
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 43
+; SI-NEXT: v_readlane_b32 s18, v22, 2
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 44, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 48
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s47, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s78, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s75, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 47
-; SI-NEXT: v_readlane_b32 s19, v21, 44
+; SI-NEXT: v_readlane_b32 s17, v21, 4
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 3
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s79, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 45
+; SI-NEXT: v_readlane_b32 s18, v22, 4
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 46
+; SI-NEXT: v_readlane_b32 s19, v22, 5
; SI-NEXT: v_add_i32_e32 v1, vcc, 48, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 47
+; SI-NEXT: v_readlane_b32 s18, v22, 6
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 48
+; SI-NEXT: v_readlane_b32 s19, v22, 7
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 49
+; SI-NEXT: v_readlane_b32 s18, v22, 8
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 52, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 50
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s45, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s74, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s74, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 49
-; SI-NEXT: v_readlane_b32 s19, v21, 50
+; SI-NEXT: v_readlane_b32 s17, v21, 3
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 9
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s75, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 51
+; SI-NEXT: v_readlane_b32 s18, v22, 10
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 52
+; SI-NEXT: v_readlane_b32 s19, v22, 11
; SI-NEXT: v_add_i32_e32 v1, vcc, 56, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 53
+; SI-NEXT: v_readlane_b32 s18, v22, 12
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 54
+; SI-NEXT: v_readlane_b32 s19, v22, 13
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 55
+; SI-NEXT: v_readlane_b32 s18, v22, 14
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 60, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 52
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s43, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s48, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s73, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 51
-; SI-NEXT: v_readlane_b32 s19, v21, 56
+; SI-NEXT: v_readlane_b32 s17, v21, 2
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s19, v22, 15
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s73, 24
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 57
+; SI-NEXT: v_readlane_b32 s18, v22, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 58
+; SI-NEXT: v_readlane_b32 s19, v22, 17
; SI-NEXT: v_add_i32_e32 v1, vcc, 64, v0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 59
+; SI-NEXT: v_readlane_b32 s18, v22, 18
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 60
+; SI-NEXT: v_readlane_b32 s19, v22, 19
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v21, 61
+; SI-NEXT: v_readlane_b32 s18, v22, 20
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x44, v0
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s17, v22, 54
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s41, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_lshl_b32 s17, s92, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s72, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 53
+; SI-NEXT: v_readlane_b32 s17, v21, 1
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, s37, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x48, v0
@@ -203357,16 +204394,16 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s16
-; SI-NEXT: v_readlane_b32 s16, v21, 63
+; SI-NEXT: v_readlane_b32 s16, v22, 22
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v22, 0
+; SI-NEXT: v_readlane_b32 s17, v22, 23
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v21, 62
+; SI-NEXT: v_readlane_b32 s19, v22, 21
; SI-NEXT: s_or_b32 s14, s14, s16
-; SI-NEXT: v_readlane_b32 s16, v22, 1
-; SI-NEXT: v_readlane_b32 s17, v22, 2
+; SI-NEXT: v_readlane_b32 s16, v22, 24
+; SI-NEXT: v_readlane_b32 s17, v22, 25
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 3
+; SI-NEXT: v_readlane_b32 s18, v22, 26
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_lshl_b32 s17, s18, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
@@ -203377,13 +204414,12 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
; SI-NEXT: s_and_b32 s14, s15, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 56
-; SI-NEXT: s_lshl_b32 s15, s15, 8
+; SI-NEXT: s_lshl_b32 s15, s87, 8
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: s_and_b32 s15, s63, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 55
+; SI-NEXT: v_readlane_b32 s15, v21, 0
+; SI-NEXT: s_and_b32 s15, s15, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_lshl_b32 s16, s16, 24
+; SI-NEXT: s_lshl_b32 s16, s72, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x50, v0
@@ -203391,15 +204427,15 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s14
-; SI-NEXT: v_readlane_b32 s14, v22, 5
+; SI-NEXT: v_readlane_b32 s14, v22, 28
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 6
+; SI-NEXT: v_readlane_b32 s15, v22, 29
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: v_readlane_b32 s14, v22, 7
-; SI-NEXT: v_readlane_b32 s15, v22, 8
+; SI-NEXT: v_readlane_b32 s14, v22, 30
+; SI-NEXT: v_readlane_b32 s15, v22, 31
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 9
+; SI-NEXT: v_readlane_b32 s16, v22, 32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s16, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
@@ -203410,13 +204446,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: s_and_b32 s12, s13, 0xff
-; SI-NEXT: v_readlane_b32 s13, v22, 58
-; SI-NEXT: s_lshl_b32 s13, s13, 8
+; SI-NEXT: s_lshl_b32 s13, s31, 8
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: s_and_b32 s13, s62, 0xff
-; SI-NEXT: v_readlane_b32 s14, v22, 57
+; SI-NEXT: s_and_b32 s13, s97, 0xff
; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s14, s14, 24
+; SI-NEXT: s_lshl_b32 s14, s84, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x58, v0
@@ -203424,15 +204458,15 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s12
-; SI-NEXT: v_readlane_b32 s12, v22, 11
+; SI-NEXT: v_readlane_b32 s12, v22, 34
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v22, 12
+; SI-NEXT: v_readlane_b32 s13, v22, 35
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: v_readlane_b32 s12, v22, 13
-; SI-NEXT: v_readlane_b32 s13, v22, 14
+; SI-NEXT: v_readlane_b32 s12, v22, 36
+; SI-NEXT: v_readlane_b32 s13, v22, 37
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v22, 15
+; SI-NEXT: v_readlane_b32 s14, v22, 38
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_lshl_b32 s13, s14, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
@@ -203443,13 +204477,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: s_and_b32 s10, s11, 0xff
-; SI-NEXT: v_readlane_b32 s11, v22, 60
-; SI-NEXT: s_lshl_b32 s11, s11, 8
+; SI-NEXT: s_lshl_b32 s11, s70, 8
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_and_b32 s11, s61, 0xff
-; SI-NEXT: v_readlane_b32 s12, v22, 59
+; SI-NEXT: s_and_b32 s11, s53, 0xff
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s12, s12, 24
+; SI-NEXT: s_lshl_b32 s12, s63, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x60, v0
@@ -203457,15 +204489,15 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_readlane_b32 s10, v22, 17
+; SI-NEXT: v_readlane_b32 s10, v22, 40
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v22, 18
+; SI-NEXT: v_readlane_b32 s11, v22, 41
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: v_readlane_b32 s10, v22, 19
-; SI-NEXT: v_readlane_b32 s11, v22, 20
+; SI-NEXT: v_readlane_b32 s10, v22, 42
+; SI-NEXT: v_readlane_b32 s11, v22, 43
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v22, 21
+; SI-NEXT: v_readlane_b32 s12, v22, 44
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_lshl_b32 s11, s12, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
@@ -203476,13 +204508,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: s_and_b32 s8, s9, 0xff
-; SI-NEXT: v_readlane_b32 s9, v22, 62
-; SI-NEXT: s_lshl_b32 s9, s9, 8
+; SI-NEXT: s_lshl_b32 s9, s67, 8
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_and_b32 s9, s60, 0xff
-; SI-NEXT: v_readlane_b32 s10, v22, 61
+; SI-NEXT: s_and_b32 s9, s61, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s10, s10, 24
+; SI-NEXT: s_lshl_b32 s10, s62, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x68, v0
@@ -203490,17 +204520,16 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_readlane_b32 s8, v22, 23
+; SI-NEXT: v_readlane_b32 s8, v22, 46
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v22, 24
+; SI-NEXT: v_readlane_b32 s9, v22, 47
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: v_readlane_b32 s8, v22, 25
-; SI-NEXT: v_readlane_b32 s9, v22, 26
+; SI-NEXT: v_readlane_b32 s8, v22, 48
+; SI-NEXT: v_readlane_b32 s9, v22, 49
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v22, 27
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s58, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x6c, v0
@@ -203509,13 +204538,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_and_b32 s6, s7, 0xff
-; SI-NEXT: v_readlane_b32 s7, v23, 0
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_lshl_b32 s7, s94, 8
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_and_b32 s7, s59, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 63
+; SI-NEXT: s_and_b32 s7, s51, 0xff
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_lshl_b32 s8, s54, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x70, v0
@@ -203523,17 +204550,12 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: v_readlane_b32 s6, v22, 29
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s7, v22, 30
-; SI-NEXT: s_lshl_b32 s6, s6, 8
+; SI-NEXT: s_lshl_b32 s6, s52, 8
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: v_readlane_b32 s6, v22, 31
-; SI-NEXT: v_readlane_b32 s7, v22, 32
-; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 33
+; SI-NEXT: s_and_b32 s6, s50, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s96, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x74, v0
@@ -203542,12 +204564,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_and_b32 s4, s5, 0xff
-; SI-NEXT: s_lshl_b32 s5, s52, 8
+; SI-NEXT: s_lshl_b32 s5, s99, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s58, 0xff
-; SI-NEXT: v_readlane_b32 s6, v23, 1
+; SI-NEXT: s_and_b32 s5, s59, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_lshl_b32 s6, s60, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x78, v0
@@ -203556,12 +204577,10 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x7c, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v22, 4
-; SI-NEXT: v_readlane_b32 s17, v22, 10
-; SI-NEXT: v_readlane_b32 s15, v22, 16
-; SI-NEXT: v_readlane_b32 s13, v22, 22
-; SI-NEXT: v_readlane_b32 s11, v22, 28
-; SI-NEXT: v_readlane_b32 s9, v22, 34
+; SI-NEXT: v_readlane_b32 s19, v22, 27
+; SI-NEXT: v_readlane_b32 s17, v22, 33
+; SI-NEXT: v_readlane_b32 s15, v22, 39
+; SI-NEXT: v_readlane_b32 s13, v22, 45
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -203602,236 +204621,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB99_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: v_writelane_b32 v21, s4, 3
-; SI-NEXT: v_writelane_b32 v21, s5, 4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v21, s4, 5
-; SI-NEXT: v_writelane_b32 v21, s5, 6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 7
-; SI-NEXT: v_writelane_b32 v21, s5, 8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 9
-; SI-NEXT: v_writelane_b32 v21, s5, 10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 11
-; SI-NEXT: v_writelane_b32 v21, s5, 12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 13
-; SI-NEXT: v_writelane_b32 v21, s5, 14
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 15
-; SI-NEXT: v_writelane_b32 v21, s5, 16
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 17
-; SI-NEXT: v_writelane_b32 v21, s5, 18
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 19
-; SI-NEXT: v_writelane_b32 v21, s5, 20
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 21
-; SI-NEXT: v_writelane_b32 v21, s5, 22
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 23
-; SI-NEXT: v_writelane_b32 v21, s5, 24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 25
-; SI-NEXT: v_writelane_b32 v21, s5, 26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 27
-; SI-NEXT: v_writelane_b32 v21, s5, 28
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 29
-; SI-NEXT: v_writelane_b32 v21, s5, 30
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 31
-; SI-NEXT: v_writelane_b32 v21, s5, 32
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 33
-; SI-NEXT: v_writelane_b32 v21, s5, 34
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 35
-; SI-NEXT: v_writelane_b32 v21, s5, 36
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 37
-; SI-NEXT: v_writelane_b32 v21, s5, 38
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 39
-; SI-NEXT: v_writelane_b32 v21, s5, 40
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 41
-; SI-NEXT: v_writelane_b32 v21, s5, 42
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 43
-; SI-NEXT: v_writelane_b32 v21, s5, 44
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 45
-; SI-NEXT: v_writelane_b32 v21, s5, 46
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 47
-; SI-NEXT: v_writelane_b32 v21, s5, 48
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 49
-; SI-NEXT: v_writelane_b32 v21, s5, 50
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 51
-; SI-NEXT: v_writelane_b32 v21, s5, 52
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 53
-; SI-NEXT: v_writelane_b32 v21, s5, 54
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 55
-; SI-NEXT: v_writelane_b32 v21, s5, 56
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 57
-; SI-NEXT: v_writelane_b32 v21, s5, 58
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 59
-; SI-NEXT: v_writelane_b32 v21, s5, 60
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 61
-; SI-NEXT: v_writelane_b32 v21, s5, 62
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v21, s4, 63
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_writelane_b32 v22, s5, 0
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 1
-; SI-NEXT: v_writelane_b32 v22, s5, 2
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 3
-; SI-NEXT: v_writelane_b32 v22, s5, 4
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 5
-; SI-NEXT: v_writelane_b32 v22, s5, 6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 7
-; SI-NEXT: v_writelane_b32 v22, s5, 8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 9
-; SI-NEXT: v_writelane_b32 v22, s5, 10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 11
-; SI-NEXT: v_writelane_b32 v22, s5, 12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 13
-; SI-NEXT: v_writelane_b32 v22, s5, 14
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 15
-; SI-NEXT: v_writelane_b32 v22, s5, 16
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 17
-; SI-NEXT: v_writelane_b32 v22, s5, 18
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 19
-; SI-NEXT: v_writelane_b32 v22, s5, 20
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 21
-; SI-NEXT: v_writelane_b32 v22, s5, 22
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 23
-; SI-NEXT: v_writelane_b32 v22, s5, 24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 25
-; SI-NEXT: v_writelane_b32 v22, s5, 26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s4, 27
-; SI-NEXT: v_writelane_b32 v22, s5, 28
-; SI-NEXT: v_writelane_b32 v22, s52, 29
-; SI-NEXT: v_writelane_b32 v22, s53, 30
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v22, s52, 31
-; SI-NEXT: v_writelane_b32 v22, s53, 32
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: v_writelane_b32 v22, s52, 33
-; SI-NEXT: v_writelane_b32 v22, s53, 34
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: s_branch .LBB99_2
;
; VI-LABEL: bitcast_v64i16_to_v128i8_scalar:
; VI: ; %bb.0:
@@ -203893,164 +204685,345 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: s_cmp_lg_u32 s44, 0
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_cbranch_scc0 .LBB99_4
+; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 0
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 1
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s29, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s29, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s28, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 8
; VI-NEXT: s_lshr_b32 s46, s28, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s27, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 11
-; VI-NEXT: s_lshr_b32 s46, s27, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s26, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 13
-; VI-NEXT: s_lshr_b32 s46, s26, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s25, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s23, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s23, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s21, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s20, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s19, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s19, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s18, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s17, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s17, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s16, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s5, 24
; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s5, 16
+; VI-NEXT: s_lshr_b32 s46, s27, 8
; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s5, 8
+; VI-NEXT: s_lshr_b32 s46, s26, 8
; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s4, 16
+; VI-NEXT: s_lshr_b32 s46, s25, 8
; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s4, 8
+; VI-NEXT: s_lshr_b32 s46, s24, 8
; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s7, 24
+; VI-NEXT: s_lshr_b32 s46, s23, 8
; VI-NEXT: v_writelane_b32 v33, s46, 35
-; VI-NEXT: s_lshr_b32 s46, s7, 16
+; VI-NEXT: s_lshr_b32 s46, s22, 8
; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s7, 8
+; VI-NEXT: s_lshr_b32 s46, s21, 8
; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s6, 16
+; VI-NEXT: s_lshr_b32 s46, s20, 8
; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s6, 8
+; VI-NEXT: s_lshr_b32 s46, s19, 8
; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s9, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s8, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s11, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s11, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s10, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s10, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s13, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s12, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s15, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s15, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s14, 16
-; VI-NEXT: s_lshr_b32 s71, s25, 8
-; VI-NEXT: s_lshr_b32 s87, s24, 8
-; VI-NEXT: s_lshr_b32 s86, s23, 8
-; VI-NEXT: s_lshr_b32 s64, s22, 8
-; VI-NEXT: s_lshr_b32 s66, s21, 8
-; VI-NEXT: s_lshr_b32 s54, s20, 8
-; VI-NEXT: s_lshr_b32 s82, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 8
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s68, s16, 8
-; VI-NEXT: s_lshr_b32 s69, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b32 s83, s14, 8
-; VI-NEXT: s_lshr_b32 s84, s41, 24
-; VI-NEXT: s_lshr_b32 s80, s41, 16
-; VI-NEXT: s_lshr_b32 s51, s41, 8
-; VI-NEXT: s_lshr_b32 s50, s40, 16
-; VI-NEXT: s_lshr_b32 s85, s40, 8
-; VI-NEXT: s_lshr_b32 s55, s43, 24
-; VI-NEXT: s_lshr_b32 s65, s43, 16
-; VI-NEXT: s_lshr_b32 s52, s43, 8
-; VI-NEXT: s_lshr_b32 s53, s42, 16
-; VI-NEXT: s_lshr_b32 s70, s42, 8
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
+; VI-NEXT: s_lshr_b32 s46, s17, 16
+; VI-NEXT: v_writelane_b32 v33, s46, 28
+; VI-NEXT: s_lshr_b32 s46, s16, 16
+; VI-NEXT: v_writelane_b32 v33, s46, 29
+; VI-NEXT: s_lshr_b32 s47, s10, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 40
+; VI-NEXT: s_lshr_b32 s47, s13, 24
+; VI-NEXT: v_writelane_b32 v33, s47, 41
+; VI-NEXT: s_lshr_b32 s47, s13, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 42
+; VI-NEXT: s_lshr_b32 s47, s13, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 43
+; VI-NEXT: s_lshr_b32 s47, s12, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 44
+; VI-NEXT: s_lshr_b32 s47, s12, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 45
+; VI-NEXT: s_lshr_b32 s47, s15, 24
+; VI-NEXT: v_writelane_b32 v33, s47, 46
+; VI-NEXT: s_lshr_b32 s47, s15, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 47
+; VI-NEXT: s_lshr_b32 s47, s15, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 48
+; VI-NEXT: s_lshr_b32 s47, s14, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 49
+; VI-NEXT: s_lshr_b32 s47, s14, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 50
+; VI-NEXT: s_lshr_b32 s47, s41, 24
+; VI-NEXT: v_writelane_b32 v33, s47, 51
+; VI-NEXT: s_lshr_b32 s47, s41, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 52
+; VI-NEXT: s_lshr_b32 s47, s41, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 53
+; VI-NEXT: s_lshr_b32 s47, s40, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 54
+; VI-NEXT: s_lshr_b32 s47, s40, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 55
+; VI-NEXT: s_lshr_b32 s47, s43, 24
+; VI-NEXT: v_writelane_b32 v33, s47, 56
+; VI-NEXT: s_lshr_b32 s47, s43, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 57
+; VI-NEXT: s_lshr_b32 s47, s43, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 58
+; VI-NEXT: s_lshr_b32 s47, s42, 16
+; VI-NEXT: v_writelane_b32 v33, s47, 59
+; VI-NEXT: s_lshr_b32 s47, s42, 8
+; VI-NEXT: v_writelane_b32 v33, s47, 60
+; VI-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 14
+; VI-NEXT: v_writelane_b32 v33, s35, 15
+; VI-NEXT: s_lshr_b64 s[34:35], s[28:29], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 12
+; VI-NEXT: v_writelane_b32 v33, s35, 13
+; VI-NEXT: s_lshr_b64 s[34:35], s[26:27], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 10
+; VI-NEXT: v_writelane_b32 v33, s35, 11
+; VI-NEXT: s_lshr_b64 s[34:35], s[24:25], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 8
+; VI-NEXT: v_writelane_b32 v33, s35, 9
; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: s_cbranch_execnz .LBB99_3
-; VI-NEXT: .LBB99_2: ; %cmp.true
+; VI-NEXT: v_writelane_b32 v33, s34, 6
+; VI-NEXT: v_writelane_b32 v33, s35, 7
+; VI-NEXT: s_lshr_b64 s[34:35], s[20:21], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 4
+; VI-NEXT: v_writelane_b32 v33, s35, 5
+; VI-NEXT: s_lshr_b64 s[34:35], s[18:19], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 2
+; VI-NEXT: v_writelane_b32 v33, s35, 3
+; VI-NEXT: s_lshr_b64 s[34:35], s[16:17], 24
+; VI-NEXT: v_writelane_b32 v33, s34, 0
+; VI-NEXT: v_writelane_b32 v33, s35, 1
+; VI-NEXT: s_lshr_b64 s[48:49], s[4:5], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 26
+; VI-NEXT: v_writelane_b32 v33, s49, 27
+; VI-NEXT: s_lshr_b64 s[48:49], s[8:9], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 24
+; VI-NEXT: v_writelane_b32 v33, s49, 25
+; VI-NEXT: s_lshr_b64 s[48:49], s[10:11], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 22
+; VI-NEXT: v_writelane_b32 v33, s49, 23
+; VI-NEXT: s_lshr_b64 s[48:49], s[14:15], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 20
+; VI-NEXT: v_writelane_b32 v33, s49, 21
+; VI-NEXT: s_lshr_b64 s[48:49], s[40:41], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 18
+; VI-NEXT: v_writelane_b32 v33, s49, 19
+; VI-NEXT: s_lshr_b64 s[48:49], s[42:43], 24
+; VI-NEXT: v_writelane_b32 v33, s48, 16
+; VI-NEXT: s_lshr_b32 s69, s45, 24
+; VI-NEXT: s_lshr_b32 s71, s45, 16
+; VI-NEXT: s_lshr_b32 s80, s45, 8
+; VI-NEXT: s_lshr_b32 s87, s44, 16
+; VI-NEXT: s_lshr_b32 s83, s44, 8
+; VI-NEXT: s_lshr_b32 s84, s29, 24
+; VI-NEXT: s_lshr_b32 s50, s29, 16
+; VI-NEXT: s_lshr_b32 s89, s29, 8
+; VI-NEXT: s_lshr_b32 s51, s28, 16
+; VI-NEXT: s_lshr_b32 s86, s27, 24
+; VI-NEXT: s_lshr_b32 s65, s27, 16
+; VI-NEXT: s_lshr_b32 s55, s26, 16
+; VI-NEXT: s_lshr_b32 s52, s25, 24
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s70, s24, 16
+; VI-NEXT: s_lshr_b32 s56, s23, 24
+; VI-NEXT: s_lshr_b32 s57, s23, 16
+; VI-NEXT: s_lshr_b32 s58, s22, 16
+; VI-NEXT: s_lshr_b32 s59, s21, 24
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s62, s20, 16
+; VI-NEXT: s_lshr_b32 s63, s19, 24
+; VI-NEXT: s_lshr_b32 s73, s19, 16
+; VI-NEXT: s_lshr_b32 s75, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s18, 8
+; VI-NEXT: s_lshr_b32 s77, s17, 24
+; VI-NEXT: s_lshr_b32 s31, s17, 8
+; VI-NEXT: s_lshr_b32 s36, s16, 8
+; VI-NEXT: s_lshr_b32 s85, s5, 24
+; VI-NEXT: s_lshr_b32 s53, s5, 16
+; VI-NEXT: s_lshr_b32 s64, s5, 8
+; VI-NEXT: s_lshr_b32 s54, s4, 16
+; VI-NEXT: s_lshr_b32 s46, s4, 8
+; VI-NEXT: s_lshr_b32 s82, s7, 24
+; VI-NEXT: s_lshr_b32 s81, s7, 16
+; VI-NEXT: s_lshr_b32 s67, s7, 8
+; VI-NEXT: s_lshr_b32 s61, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s6, 8
+; VI-NEXT: s_lshr_b32 s76, s9, 24
+; VI-NEXT: s_lshr_b32 s72, s9, 16
+; VI-NEXT: s_lshr_b32 s74, s9, 8
+; VI-NEXT: s_lshr_b32 s79, s8, 16
+; VI-NEXT: s_lshr_b32 s78, s8, 8
+; VI-NEXT: s_lshr_b32 s90, s11, 24
+; VI-NEXT: s_lshr_b32 s88, s11, 16
+; VI-NEXT: s_lshr_b32 s91, s11, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s10, 16
+; VI-NEXT: s_lshr_b64 s[34:35], s[6:7], 24
+; VI-NEXT: s_lshr_b64 s[38:39], s[12:13], 24
+; VI-NEXT: v_writelane_b32 v33, s49, 17
+; VI-NEXT: s_mov_b64 s[48:49], 0
+; VI-NEXT: s_branch .LBB99_3
+; VI-NEXT: .LBB99_2:
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: s_mov_b64 s[48:49], -1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 0
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 1
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr68
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr70
+; VI-NEXT: ; implicit-def: $sgpr66
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr86
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr84
+; VI-NEXT: ; implicit-def: $sgpr83
+; VI-NEXT: ; implicit-def: $sgpr87
+; VI-NEXT: ; implicit-def: $sgpr80
+; VI-NEXT: ; implicit-def: $sgpr71
+; VI-NEXT: ; implicit-def: $sgpr69
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr67
+; VI-NEXT: ; implicit-def: $sgpr81
+; VI-NEXT: ; implicit-def: $sgpr82
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr85
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 2
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 3
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 4
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 5
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 6
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 7
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 8
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 9
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 10
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s31, 11
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 12
+; VI-NEXT: v_writelane_b32 v33, s31, 13
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 14
+; VI-NEXT: v_writelane_b32 v33, s31, 15
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 16
+; VI-NEXT: v_writelane_b32 v33, s31, 17
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 18
+; VI-NEXT: v_writelane_b32 v33, s31, 19
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 20
+; VI-NEXT: v_writelane_b32 v33, s31, 21
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 22
+; VI-NEXT: v_writelane_b32 v33, s31, 23
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 24
+; VI-NEXT: v_writelane_b32 v33, s31, 25
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; kill: killed $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: v_writelane_b32 v33, s30, 26
+; VI-NEXT: v_writelane_b32 v33, s31, 27
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: .LBB99_3: ; %Flow
+; VI-NEXT: s_and_b64 s[48:49], s[48:49], exec
+; VI-NEXT: s_cselect_b32 s47, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s47, 1
+; VI-NEXT: s_mov_b32 s48, s85
+; VI-NEXT: s_mov_b32 s49, s53
+; VI-NEXT: s_mov_b32 s53, s54
+; VI-NEXT: s_mov_b32 s47, s82
+; VI-NEXT: s_mov_b32 s54, s81
+; VI-NEXT: s_mov_b32 s81, s67
+; VI-NEXT: s_mov_b32 s67, s61
+; VI-NEXT: s_mov_b32 s61, s76
+; VI-NEXT: s_mov_b32 s76, s79
+; VI-NEXT: v_readlane_b32 s79, v33, 28
+; VI-NEXT: v_readlane_b32 s82, v33, 29
+; VI-NEXT: s_mov_b32 s85, s90
+; VI-NEXT: s_mov_b32 s90, vcc_lo
+; VI-NEXT: s_cbranch_scc1 .LBB99_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s46, s43, 0xffff0000
; VI-NEXT: s_add_i32 s43, s43, 3
; VI-NEXT: s_and_b32 s43, s43, 0xffff
@@ -204177,509 +205150,515 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: s_or_b32 s45, s46, s45
; VI-NEXT: s_and_b32 s46, s44, 0xffff0000
; VI-NEXT: s_add_i32 s44, s44, 3
-; VI-NEXT: s_add_i32 s45, s45, 0x30000
-; VI-NEXT: s_and_b32 s44, s44, 0xffff
-; VI-NEXT: s_or_b32 s44, s46, s44
-; VI-NEXT: s_lshr_b32 s46, s45, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 0
-; VI-NEXT: s_lshr_b32 s46, s45, 16
-; VI-NEXT: s_add_i32 s44, s44, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 1
-; VI-NEXT: s_lshr_b32 s46, s45, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 2
-; VI-NEXT: s_lshr_b32 s46, s44, 16
-; VI-NEXT: s_add_i32 s29, s29, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 3
-; VI-NEXT: s_lshr_b32 s46, s44, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 4
-; VI-NEXT: s_lshr_b32 s46, s29, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 5
-; VI-NEXT: s_lshr_b32 s46, s29, 16
; VI-NEXT: s_add_i32 s28, s28, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 6
-; VI-NEXT: s_lshr_b32 s46, s29, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 7
-; VI-NEXT: s_lshr_b32 s46, s28, 16
+; VI-NEXT: s_and_b32 s44, s44, 0xffff
; VI-NEXT: s_add_i32 s27, s27, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 8
+; VI-NEXT: s_or_b32 s44, s46, s44
; VI-NEXT: s_lshr_b32 s46, s28, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 9
-; VI-NEXT: s_lshr_b32 s46, s27, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 10
-; VI-NEXT: s_lshr_b32 s46, s27, 16
; VI-NEXT: s_add_i32 s26, s26, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 11
+; VI-NEXT: v_writelane_b32 v33, s46, 30
; VI-NEXT: s_lshr_b32 s46, s27, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 12
-; VI-NEXT: s_lshr_b32 s46, s26, 16
; VI-NEXT: s_add_i32 s25, s25, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 13
+; VI-NEXT: v_writelane_b32 v33, s46, 31
; VI-NEXT: s_lshr_b32 s46, s26, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 14
-; VI-NEXT: s_lshr_b32 s46, s25, 24
; VI-NEXT: s_add_i32 s24, s24, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 15
-; VI-NEXT: s_lshr_b32 s46, s25, 16
-; VI-NEXT: s_add_i32 s23, s23, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 16
-; VI-NEXT: s_lshr_b32 s46, s24, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 17
-; VI-NEXT: s_lshr_b32 s46, s23, 24
-; VI-NEXT: s_add_i32 s22, s22, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 18
-; VI-NEXT: s_lshr_b32 s46, s23, 16
-; VI-NEXT: s_add_i32 s21, s21, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 19
-; VI-NEXT: s_lshr_b32 s46, s22, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 20
-; VI-NEXT: s_lshr_b32 s46, s21, 24
-; VI-NEXT: s_add_i32 s20, s20, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 21
-; VI-NEXT: s_lshr_b32 s46, s21, 16
-; VI-NEXT: s_add_i32 s19, s19, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 22
-; VI-NEXT: s_lshr_b32 s46, s20, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 23
-; VI-NEXT: s_lshr_b32 s46, s19, 24
-; VI-NEXT: s_add_i32 s18, s18, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 24
-; VI-NEXT: s_lshr_b32 s46, s19, 16
-; VI-NEXT: s_add_i32 s17, s17, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 25
-; VI-NEXT: s_lshr_b32 s46, s18, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 26
-; VI-NEXT: s_lshr_b32 s46, s17, 24
-; VI-NEXT: s_add_i32 s16, s16, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 27
-; VI-NEXT: s_lshr_b32 s46, s17, 16
-; VI-NEXT: s_add_i32 s5, s5, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 28
-; VI-NEXT: s_lshr_b32 s46, s16, 16
-; VI-NEXT: v_writelane_b32 v33, s46, 29
-; VI-NEXT: s_lshr_b32 s46, s5, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 30
-; VI-NEXT: s_lshr_b32 s46, s5, 16
-; VI-NEXT: s_add_i32 s4, s4, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 31
-; VI-NEXT: s_lshr_b32 s46, s5, 8
; VI-NEXT: v_writelane_b32 v33, s46, 32
-; VI-NEXT: s_lshr_b32 s46, s4, 16
-; VI-NEXT: s_add_i32 s7, s7, 0x30000
+; VI-NEXT: s_lshr_b32 s46, s25, 8
+; VI-NEXT: s_add_i32 s23, s23, 0x30000
; VI-NEXT: v_writelane_b32 v33, s46, 33
-; VI-NEXT: s_lshr_b32 s46, s4, 8
+; VI-NEXT: s_lshr_b32 s46, s24, 8
+; VI-NEXT: s_add_i32 s22, s22, 0x30000
; VI-NEXT: v_writelane_b32 v33, s46, 34
-; VI-NEXT: s_lshr_b32 s46, s7, 24
+; VI-NEXT: s_lshr_b32 s46, s23, 8
+; VI-NEXT: s_add_i32 s21, s21, 0x30000
; VI-NEXT: v_writelane_b32 v33, s46, 35
-; VI-NEXT: s_lshr_b32 s46, s7, 16
-; VI-NEXT: s_add_i32 s6, s6, 0x30000
+; VI-NEXT: s_lshr_b32 s46, s22, 8
+; VI-NEXT: s_add_i32 s20, s20, 0x30000
; VI-NEXT: v_writelane_b32 v33, s46, 36
-; VI-NEXT: s_lshr_b32 s46, s7, 8
+; VI-NEXT: s_lshr_b32 s46, s21, 8
+; VI-NEXT: s_add_i32 s19, s19, 0x30000
; VI-NEXT: v_writelane_b32 v33, s46, 37
-; VI-NEXT: s_lshr_b32 s46, s6, 16
-; VI-NEXT: s_add_i32 s9, s9, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 38
-; VI-NEXT: s_lshr_b32 s46, s6, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 39
-; VI-NEXT: s_lshr_b32 s46, s9, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 40
-; VI-NEXT: s_lshr_b32 s46, s9, 16
-; VI-NEXT: s_add_i32 s8, s8, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 41
-; VI-NEXT: s_lshr_b32 s46, s9, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 42
-; VI-NEXT: s_lshr_b32 s46, s8, 16
-; VI-NEXT: s_add_i32 s11, s11, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 43
-; VI-NEXT: s_lshr_b32 s46, s8, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 44
-; VI-NEXT: s_lshr_b32 s46, s11, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 45
-; VI-NEXT: s_lshr_b32 s46, s11, 16
+; VI-NEXT: s_lshr_b32 s46, s20, 8
; VI-NEXT: s_add_i32 s10, s10, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 46
-; VI-NEXT: s_lshr_b32 s46, s11, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 47
-; VI-NEXT: s_lshr_b32 s46, s10, 16
+; VI-NEXT: v_writelane_b32 v33, s46, 38
+; VI-NEXT: s_lshr_b32 s46, s19, 8
; VI-NEXT: s_add_i32 s13, s13, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 48
-; VI-NEXT: s_lshr_b32 s46, s10, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 49
-; VI-NEXT: s_lshr_b32 s46, s13, 24
-; VI-NEXT: v_writelane_b32 v33, s46, 50
-; VI-NEXT: s_lshr_b32 s46, s13, 16
+; VI-NEXT: v_writelane_b32 v33, s46, 39
+; VI-NEXT: s_lshr_b32 vcc_lo, s10, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 40
+; VI-NEXT: s_lshr_b32 vcc_lo, s13, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 41
+; VI-NEXT: s_lshr_b32 vcc_lo, s13, 16
; VI-NEXT: s_add_i32 s12, s12, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 51
-; VI-NEXT: s_lshr_b32 s46, s13, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 52
-; VI-NEXT: s_lshr_b32 s46, s12, 16
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 42
+; VI-NEXT: s_lshr_b32 vcc_lo, s13, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 43
+; VI-NEXT: s_lshr_b32 vcc_lo, s12, 16
; VI-NEXT: s_add_i32 s15, s15, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 53
-; VI-NEXT: s_lshr_b32 s46, s12, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 54
-; VI-NEXT: s_lshr_b32 s46, s15, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 44
+; VI-NEXT: s_lshr_b32 vcc_lo, s12, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 45
+; VI-NEXT: s_lshr_b32 vcc_lo, s15, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 46
+; VI-NEXT: s_lshr_b32 vcc_lo, s15, 16
; VI-NEXT: s_add_i32 s14, s14, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 55
-; VI-NEXT: s_lshr_b32 s46, s15, 16
-; VI-NEXT: s_add_i32 s43, s43, 0x30000
-; VI-NEXT: s_add_i32 s42, s42, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 47
+; VI-NEXT: s_lshr_b32 vcc_lo, s15, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 48
+; VI-NEXT: s_lshr_b32 vcc_lo, s14, 16
; VI-NEXT: s_add_i32 s41, s41, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 49
+; VI-NEXT: s_lshr_b32 vcc_lo, s14, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 50
+; VI-NEXT: s_lshr_b32 vcc_lo, s41, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 51
+; VI-NEXT: s_lshr_b32 vcc_lo, s41, 16
; VI-NEXT: s_add_i32 s40, s40, 0x30000
-; VI-NEXT: v_writelane_b32 v33, s46, 56
-; VI-NEXT: s_lshr_b32 s46, s14, 16
-; VI-NEXT: s_lshr_b32 s71, s25, 8
-; VI-NEXT: s_lshr_b32 s87, s24, 8
-; VI-NEXT: s_lshr_b32 s86, s23, 8
-; VI-NEXT: s_lshr_b32 s64, s22, 8
-; VI-NEXT: s_lshr_b32 s66, s21, 8
-; VI-NEXT: s_lshr_b32 s54, s20, 8
-; VI-NEXT: s_lshr_b32 s82, s19, 8
-; VI-NEXT: s_lshr_b32 s81, s18, 8
-; VI-NEXT: s_lshr_b32 s67, s17, 8
-; VI-NEXT: s_lshr_b32 s68, s16, 8
-; VI-NEXT: s_lshr_b32 s69, s15, 8
-; VI-NEXT: v_writelane_b32 v33, s46, 57
-; VI-NEXT: s_lshr_b32 s83, s14, 8
-; VI-NEXT: s_lshr_b32 s84, s41, 24
-; VI-NEXT: s_lshr_b32 s80, s41, 16
-; VI-NEXT: s_lshr_b32 s51, s41, 8
-; VI-NEXT: s_lshr_b32 s50, s40, 16
-; VI-NEXT: s_lshr_b32 s85, s40, 8
-; VI-NEXT: s_lshr_b32 s55, s43, 24
-; VI-NEXT: s_lshr_b32 s65, s43, 16
-; VI-NEXT: s_lshr_b32 s52, s43, 8
-; VI-NEXT: s_lshr_b32 s53, s42, 16
-; VI-NEXT: s_lshr_b32 s70, s42, 8
-; VI-NEXT: s_lshr_b64 s[78:79], s[44:45], 24
-; VI-NEXT: s_lshr_b64 s[88:89], s[28:29], 24
-; VI-NEXT: s_lshr_b64 s[90:91], s[26:27], 24
-; VI-NEXT: s_lshr_b64 s[30:31], s[24:25], 24
-; VI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; VI-NEXT: s_lshr_b64 s[36:37], s[20:21], 24
-; VI-NEXT: s_lshr_b64 s[38:39], s[18:19], 24
-; VI-NEXT: s_lshr_b64 s[48:49], s[16:17], 24
-; VI-NEXT: s_lshr_b64 s[46:47], s[4:5], 24
-; VI-NEXT: s_lshr_b64 s[56:57], s[6:7], 24
-; VI-NEXT: s_lshr_b64 s[58:59], s[8:9], 24
-; VI-NEXT: s_lshr_b64 s[60:61], s[10:11], 24
-; VI-NEXT: s_lshr_b64 s[62:63], s[12:13], 24
-; VI-NEXT: s_lshr_b64 s[72:73], s[14:15], 24
-; VI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
-; VI-NEXT: s_lshr_b64 s[76:77], s[42:43], 24
-; VI-NEXT: .LBB99_3: ; %end
-; VI-NEXT: v_mov_b32_e32 v2, s68
-; VI-NEXT: v_mov_b32_e32 v4, s67
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 52
+; VI-NEXT: s_lshr_b32 vcc_lo, s41, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 53
+; VI-NEXT: s_lshr_b32 vcc_lo, s40, 16
+; VI-NEXT: s_add_i32 s43, s43, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 54
+; VI-NEXT: s_lshr_b32 vcc_lo, s40, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 55
+; VI-NEXT: s_lshr_b32 vcc_lo, s43, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 56
+; VI-NEXT: s_lshr_b32 vcc_lo, s43, 16
+; VI-NEXT: s_add_i32 s42, s42, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 57
+; VI-NEXT: s_lshr_b32 vcc_lo, s43, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 58
+; VI-NEXT: s_lshr_b32 vcc_lo, s42, 16
+; VI-NEXT: s_add_i32 s45, s45, 0x30000
+; VI-NEXT: s_add_i32 s44, s44, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 59
+; VI-NEXT: s_lshr_b32 vcc_lo, s42, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 60
+; VI-NEXT: s_lshr_b64 vcc, s[44:45], 24
+; VI-NEXT: s_add_i32 s29, s29, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 14
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 15
+; VI-NEXT: s_lshr_b64 vcc, s[28:29], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 12
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 13
+; VI-NEXT: s_lshr_b64 vcc, s[26:27], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 10
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 11
+; VI-NEXT: s_lshr_b64 vcc, s[24:25], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 9
+; VI-NEXT: s_lshr_b64 vcc, s[22:23], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 6
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 7
+; VI-NEXT: s_lshr_b64 vcc, s[20:21], 24
+; VI-NEXT: s_add_i32 s18, s18, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 4
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 5
+; VI-NEXT: s_lshr_b64 vcc, s[18:19], 24
+; VI-NEXT: s_add_i32 s17, s17, 0x30000
+; VI-NEXT: s_add_i32 s16, s16, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 2
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 3
+; VI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; VI-NEXT: s_add_i32 s5, s5, 0x30000
+; VI-NEXT: s_add_i32 s4, s4, 0x30000
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 0
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 1
+; VI-NEXT: s_lshr_b64 s[30:31], s[4:5], 24
+; VI-NEXT: s_add_i32 s9, s9, 0x30000
+; VI-NEXT: s_add_i32 s8, s8, 0x30000
+; VI-NEXT: v_writelane_b32 v33, s30, 26
+; VI-NEXT: v_writelane_b32 v33, s31, 27
+; VI-NEXT: s_lshr_b64 vcc, s[8:9], 24
+; VI-NEXT: s_add_i32 s11, s11, 0x30000
+; VI-NEXT: s_lshr_b32 s36, s17, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 24
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 25
+; VI-NEXT: s_mov_b32 s31, s36
+; VI-NEXT: s_lshr_b64 s[36:37], s[10:11], 24
+; VI-NEXT: s_lshr_b32 s38, s16, 8
+; VI-NEXT: v_writelane_b32 v33, s36, 22
+; VI-NEXT: v_writelane_b32 v33, s37, 23
+; VI-NEXT: s_mov_b32 s36, s38
+; VI-NEXT: s_lshr_b64 s[38:39], s[14:15], 24
+; VI-NEXT: s_lshr_b64 vcc, s[12:13], 24
+; VI-NEXT: v_writelane_b32 v33, s38, 20
+; VI-NEXT: v_writelane_b32 v33, s39, 21
+; VI-NEXT: s_mov_b32 s38, vcc_lo
+; VI-NEXT: s_lshr_b64 vcc, s[40:41], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 18
+; VI-NEXT: s_add_i32 s7, s7, 0x30000
+; VI-NEXT: s_add_i32 s6, s6, 0x30000
+; VI-NEXT: s_lshr_b32 s35, s18, 8
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 19
+; VI-NEXT: s_lshr_b64 vcc, s[42:43], 24
+; VI-NEXT: s_lshr_b32 s69, s45, 24
+; VI-NEXT: s_lshr_b32 s71, s45, 16
+; VI-NEXT: s_lshr_b32 s80, s45, 8
+; VI-NEXT: s_lshr_b32 s87, s44, 16
+; VI-NEXT: s_lshr_b32 s83, s44, 8
+; VI-NEXT: s_lshr_b32 s84, s29, 24
+; VI-NEXT: s_lshr_b32 s50, s29, 16
+; VI-NEXT: s_lshr_b32 s89, s29, 8
+; VI-NEXT: s_lshr_b32 s51, s28, 16
+; VI-NEXT: s_lshr_b32 s86, s27, 24
+; VI-NEXT: s_lshr_b32 s65, s27, 16
+; VI-NEXT: s_lshr_b32 s55, s26, 16
+; VI-NEXT: s_lshr_b32 s52, s25, 24
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s70, s24, 16
+; VI-NEXT: s_lshr_b32 s56, s23, 24
+; VI-NEXT: s_lshr_b32 s57, s23, 16
+; VI-NEXT: s_lshr_b32 s58, s22, 16
+; VI-NEXT: s_lshr_b32 s59, s21, 24
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s62, s20, 16
+; VI-NEXT: s_lshr_b32 s63, s19, 24
+; VI-NEXT: s_lshr_b32 s73, s19, 16
+; VI-NEXT: s_lshr_b32 s75, s18, 16
+; VI-NEXT: s_lshr_b32 s77, s17, 24
+; VI-NEXT: s_lshr_b32 s79, s17, 16
+; VI-NEXT: s_lshr_b32 s82, s16, 16
+; VI-NEXT: s_lshr_b32 s48, s5, 24
+; VI-NEXT: s_lshr_b32 s49, s5, 16
+; VI-NEXT: s_lshr_b32 s64, s5, 8
+; VI-NEXT: s_lshr_b32 s53, s4, 16
+; VI-NEXT: s_lshr_b32 s46, s4, 8
+; VI-NEXT: s_lshr_b32 s47, s7, 24
+; VI-NEXT: s_lshr_b32 s54, s7, 16
+; VI-NEXT: s_lshr_b32 s81, s7, 8
+; VI-NEXT: s_lshr_b32 s67, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s6, 8
+; VI-NEXT: s_lshr_b32 s61, s9, 24
+; VI-NEXT: s_lshr_b32 s72, s9, 16
+; VI-NEXT: s_lshr_b32 s74, s9, 8
+; VI-NEXT: s_lshr_b32 s76, s8, 16
+; VI-NEXT: s_lshr_b32 s78, s8, 8
+; VI-NEXT: s_lshr_b32 s85, s11, 24
+; VI-NEXT: s_lshr_b32 s88, s11, 16
+; VI-NEXT: s_lshr_b32 s91, s11, 8
+; VI-NEXT: s_lshr_b32 s90, s10, 16
+; VI-NEXT: s_mov_b32 s30, s35
+; VI-NEXT: s_lshr_b64 s[34:35], s[6:7], 24
+; VI-NEXT: v_writelane_b32 v33, vcc_lo, 16
+; VI-NEXT: v_writelane_b32 v33, vcc_hi, 17
+; VI-NEXT: .LBB99_5: ; %end
+; VI-NEXT: v_readlane_b32 vcc_lo, v33, 0
+; VI-NEXT: v_mov_b32_e32 v6, vcc_lo
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
+; VI-NEXT: v_mov_b32_e32 v2, s36
+; VI-NEXT: v_perm_b32 v6, s82, v6, v1
+; VI-NEXT: v_mov_b32_e32 v4, s31
; VI-NEXT: v_perm_b32 v3, s16, v2, v1
+; VI-NEXT: v_mov_b32_e32 v24, s77
+; VI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; VI-NEXT: v_perm_b32 v2, s17, v4, v1
-; VI-NEXT: v_mov_b32_e32 v4, s81
-; VI-NEXT: v_mov_b32_e32 v6, s82
-; VI-NEXT: v_perm_b32 v5, s18, v4, v1
-; VI-NEXT: v_perm_b32 v4, s19, v6, v1
-; VI-NEXT: v_mov_b32_e32 v6, s54
-; VI-NEXT: v_mov_b32_e32 v8, s66
-; VI-NEXT: v_perm_b32 v7, s20, v6, v1
-; VI-NEXT: v_perm_b32 v6, s21, v8, v1
-; VI-NEXT: v_mov_b32_e32 v8, s64
-; VI-NEXT: v_mov_b32_e32 v11, s86
-; VI-NEXT: v_perm_b32 v9, s22, v8, v1
-; VI-NEXT: v_perm_b32 v8, s23, v11, v1
-; VI-NEXT: v_mov_b32_e32 v11, s87
-; VI-NEXT: v_mov_b32_e32 v14, s71
-; VI-NEXT: v_readlane_b32 s16, v33, 14
-; VI-NEXT: v_perm_b32 v12, s24, v11, v1
-; VI-NEXT: v_perm_b32 v11, s25, v14, v1
-; VI-NEXT: v_mov_b32_e32 v14, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 12
-; VI-NEXT: v_mov_b32_e32 v17, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 9
-; VI-NEXT: v_perm_b32 v15, s26, v14, v1
-; VI-NEXT: v_perm_b32 v14, s27, v17, v1
-; VI-NEXT: v_mov_b32_e32 v17, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 7
-; VI-NEXT: v_mov_b32_e32 v10, s48
-; VI-NEXT: v_mov_b32_e32 v23, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 29
-; VI-NEXT: v_perm_b32 v10, s16, v10, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 27
-; VI-NEXT: v_mov_b32_e32 v24, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 28
-; VI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; VI-NEXT: v_perm_b32 v24, s16, v24, v1
-; VI-NEXT: v_or_b32_e32 v3, v3, v10
-; VI-NEXT: v_mov_b32_e32 v13, s38
-; VI-NEXT: v_readlane_b32 s16, v33, 26
+; VI-NEXT: v_readlane_b32 s16, v33, 2
+; VI-NEXT: v_perm_b32 v24, s79, v24, v1
+; VI-NEXT: v_or_b32_e32 v3, v3, v6
+; VI-NEXT: v_readlane_b32 vcc_hi, v33, 1
+; VI-NEXT: v_mov_b32_e32 v9, s16
; VI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; VI-NEXT: v_perm_b32 v13, s16, v13, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 24
+; VI-NEXT: v_mov_b32_e32 v4, s30
+; VI-NEXT: v_perm_b32 v9, s75, v9, v1
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 4, v0
-; VI-NEXT: v_mov_b32_e32 v25, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 25
+; VI-NEXT: v_readlane_b32 s17, v33, 3
+; VI-NEXT: v_readlane_b32 s16, v33, 39
+; VI-NEXT: v_perm_b32 v5, s18, v4, v1
+; VI-NEXT: v_mov_b32_e32 v25, s63
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v13
-; VI-NEXT: v_perm_b32 v25, s16, v25, v1
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
+; VI-NEXT: v_mov_b32_e32 v7, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 4
+; VI-NEXT: v_perm_b32 v25, s73, v25, v1
; VI-NEXT: v_or_b32_e32 v2, v5, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 8, v0
-; VI-NEXT: v_mov_b32_e32 v16, s36
-; VI-NEXT: v_readlane_b32 s16, v33, 23
+; VI-NEXT: v_perm_b32 v4, s19, v7, v1
+; VI-NEXT: v_mov_b32_e32 v12, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 38
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; VI-NEXT: v_perm_b32 v16, s16, v16, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 21
+; VI-NEXT: v_mov_b32_e32 v7, s16
+; VI-NEXT: v_perm_b32 v12, s62, v12, v1
; VI-NEXT: v_or_b32_e32 v2, v4, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 12, v0
-; VI-NEXT: v_mov_b32_e32 v26, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 22
+; VI-NEXT: v_readlane_b32 s17, v33, 5
+; VI-NEXT: v_readlane_b32 s16, v33, 37
+; VI-NEXT: v_perm_b32 v8, s20, v7, v1
+; VI-NEXT: v_mov_b32_e32 v26, s59
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
-; VI-NEXT: v_perm_b32 v26, s16, v26, v1
-; VI-NEXT: v_or_b32_e32 v2, v7, v2
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v12
+; VI-NEXT: v_mov_b32_e32 v10, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 6
+; VI-NEXT: v_perm_b32 v26, s68, v26, v1
+; VI-NEXT: v_or_b32_e32 v2, v8, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT: v_mov_b32_e32 v19, s34
-; VI-NEXT: v_readlane_b32 s16, v33, 20
+; VI-NEXT: v_perm_b32 v7, s21, v10, v1
+; VI-NEXT: v_mov_b32_e32 v16, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 36
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v26
-; VI-NEXT: v_perm_b32 v19, s16, v19, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 18
-; VI-NEXT: v_or_b32_e32 v2, v6, v2
+; VI-NEXT: v_mov_b32_e32 v10, s16
+; VI-NEXT: v_perm_b32 v16, s58, v16, v1
+; VI-NEXT: v_or_b32_e32 v2, v7, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 20, v0
-; VI-NEXT: v_mov_b32_e32 v27, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 19
+; VI-NEXT: v_readlane_b32 s17, v33, 7
+; VI-NEXT: v_readlane_b32 s16, v33, 35
+; VI-NEXT: v_perm_b32 v11, s22, v10, v1
+; VI-NEXT: v_mov_b32_e32 v27, s56
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v19
-; VI-NEXT: v_perm_b32 v27, s16, v27, v1
-; VI-NEXT: v_or_b32_e32 v2, v9, v2
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
+; VI-NEXT: v_mov_b32_e32 v13, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 8
+; VI-NEXT: v_perm_b32 v27, s57, v27, v1
+; VI-NEXT: v_or_b32_e32 v2, v11, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 24, v0
-; VI-NEXT: v_mov_b32_e32 v20, s30
-; VI-NEXT: v_readlane_b32 s16, v33, 17
+; VI-NEXT: v_perm_b32 v10, s23, v13, v1
+; VI-NEXT: v_mov_b32_e32 v20, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 34
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v27
-; VI-NEXT: v_perm_b32 v20, s16, v20, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 15
-; VI-NEXT: v_or_b32_e32 v2, v8, v2
+; VI-NEXT: v_mov_b32_e32 v13, s16
+; VI-NEXT: v_perm_b32 v20, s70, v20, v1
+; VI-NEXT: v_or_b32_e32 v2, v10, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 28, v0
-; VI-NEXT: v_mov_b32_e32 v28, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 16
+; VI-NEXT: v_readlane_b32 s17, v33, 9
+; VI-NEXT: v_readlane_b32 s16, v33, 33
+; VI-NEXT: v_perm_b32 v14, s24, v13, v1
+; VI-NEXT: v_mov_b32_e32 v28, s52
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v20
-; VI-NEXT: v_perm_b32 v28, s16, v28, v1
-; VI-NEXT: v_or_b32_e32 v2, v12, v2
+; VI-NEXT: v_mov_b32_e32 v15, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 10
+; VI-NEXT: v_perm_b32 v28, s66, v28, v1
+; VI-NEXT: v_or_b32_e32 v2, v14, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 32, v0
-; VI-NEXT: v_mov_b32_e32 v21, s90
-; VI-NEXT: v_readlane_b32 s16, v33, 13
+; VI-NEXT: v_perm_b32 v13, s25, v15, v1
+; VI-NEXT: v_mov_b32_e32 v21, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 32
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v28
-; VI-NEXT: v_perm_b32 v21, s16, v21, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 10
-; VI-NEXT: v_or_b32_e32 v2, v11, v2
+; VI-NEXT: v_mov_b32_e32 v15, s16
+; VI-NEXT: v_perm_b32 v21, s55, v21, v1
+; VI-NEXT: v_or_b32_e32 v2, v13, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 36, v0
-; VI-NEXT: v_mov_b32_e32 v29, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 11
+; VI-NEXT: v_readlane_b32 s17, v33, 11
+; VI-NEXT: v_readlane_b32 s16, v33, 31
+; VI-NEXT: v_perm_b32 v17, s26, v15, v1
+; VI-NEXT: v_mov_b32_e32 v29, s86
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v21
-; VI-NEXT: v_perm_b32 v29, s16, v29, v1
-; VI-NEXT: v_or_b32_e32 v2, v15, v2
+; VI-NEXT: v_mov_b32_e32 v18, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 12
+; VI-NEXT: v_perm_b32 v29, s65, v29, v1
+; VI-NEXT: v_or_b32_e32 v2, v17, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 40, v0
-; VI-NEXT: v_mov_b32_e32 v22, s88
-; VI-NEXT: v_readlane_b32 s16, v33, 8
+; VI-NEXT: v_perm_b32 v15, s27, v18, v1
+; VI-NEXT: v_mov_b32_e32 v22, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 30
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
-; VI-NEXT: v_perm_b32 v22, s16, v22, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 5
-; VI-NEXT: v_or_b32_e32 v2, v14, v2
+; VI-NEXT: v_mov_b32_e32 v18, s16
+; VI-NEXT: v_perm_b32 v22, s51, v22, v1
+; VI-NEXT: v_or_b32_e32 v2, v15, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 44, v0
-; VI-NEXT: v_perm_b32 v18, s28, v17, v1
-; VI-NEXT: v_mov_b32_e32 v30, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 6
+; VI-NEXT: v_readlane_b32 s17, v33, 13
+; VI-NEXT: v_perm_b32 v19, s28, v18, v1
+; VI-NEXT: v_mov_b32_e32 v30, s84
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v22
-; VI-NEXT: v_perm_b32 v30, s16, v30, v1
-; VI-NEXT: v_readlane_b32 s16, v33, 4
-; VI-NEXT: v_or_b32_e32 v2, v18, v2
+; VI-NEXT: v_mov_b32_e32 v23, s89
+; VI-NEXT: v_readlane_b32 s16, v33, 14
+; VI-NEXT: v_perm_b32 v30, s50, v30, v1
+; VI-NEXT: v_or_b32_e32 v2, v19, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 48, v0
-; VI-NEXT: v_perm_b32 v17, s29, v23, v1
-; VI-NEXT: v_mov_b32_e32 v23, s78
-; VI-NEXT: v_mov_b32_e32 v31, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 3
+; VI-NEXT: v_perm_b32 v18, s29, v23, v1
+; VI-NEXT: v_mov_b32_e32 v23, s16
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v30
-; VI-NEXT: v_perm_b32 v23, s16, v23, v1
-; VI-NEXT: v_or_b32_e32 v2, v17, v2
+; VI-NEXT: v_mov_b32_e32 v31, s83
+; VI-NEXT: v_perm_b32 v23, s87, v23, v1
+; VI-NEXT: v_or_b32_e32 v2, v18, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 52, v0
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_perm_b32 v2, s44, v31, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v23
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 56, v0
-; VI-NEXT: v_readlane_b32 s16, v33, 2
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 0
-; VI-NEXT: v_mov_b32_e32 v3, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 1
-; VI-NEXT: v_perm_b32 v3, s16, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s69
+; VI-NEXT: v_mov_b32_e32 v2, s80
+; VI-NEXT: v_perm_b32 v3, s71, v3, v1
; VI-NEXT: v_perm_b32 v2, s45, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_readlane_b32 s17, v33, 15
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 60, v0
+; VI-NEXT: v_readlane_b32 s16, v33, 60
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v3, s76
-; VI-NEXT: v_mov_b32_e32 v2, s70
-; VI-NEXT: v_perm_b32 v3, s53, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 16
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 59
+; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s42, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 64, v0
+; VI-NEXT: v_readlane_b32 s16, v33, 58
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v3, s55
-; VI-NEXT: v_mov_b32_e32 v2, s52
-; VI-NEXT: v_perm_b32 v3, s65, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 56
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 57
+; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s43, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_readlane_b32 s17, v33, 17
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x44, v0
+; VI-NEXT: v_readlane_b32 s16, v33, 55
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v3, s74
-; VI-NEXT: v_mov_b32_e32 v2, s85
-; VI-NEXT: v_perm_b32 v3, s50, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 18
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 54
+; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s40, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x48, v0
+; VI-NEXT: v_readlane_b32 s16, v33, 53
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v3, s84
-; VI-NEXT: v_mov_b32_e32 v2, s51
-; VI-NEXT: v_perm_b32 v3, s80, v3, v1
+; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 51
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 52
+; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT: v_readlane_b32 s17, v33, 19
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x4c, v0
+; VI-NEXT: v_readlane_b32 s16, v33, 50
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s83
+; VI-NEXT: v_mov_b32_e32 v2, s16
+; VI-NEXT: v_readlane_b32 s16, v33, 20
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s72
-; VI-NEXT: v_readlane_b32 s14, v33, 57
+; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s14, v33, 49
; VI-NEXT: v_perm_b32 v3, s14, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x50, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 55
+; VI-NEXT: v_readlane_b32 s14, v33, 48
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_readlane_b32 s14, v33, 46
; VI-NEXT: v_mov_b32_e32 v3, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 56
-; VI-NEXT: v_mov_b32_e32 v2, s69
+; VI-NEXT: v_readlane_b32 s14, v33, 47
; VI-NEXT: v_perm_b32 v3, s14, v3, v1
; VI-NEXT: v_perm_b32 v2, s15, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x54, v0
-; VI-NEXT: v_readlane_b32 s14, v33, 54
+; VI-NEXT: v_readlane_b32 s14, v33, 45
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_mov_b32_e32 v2, s14
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s62
-; VI-NEXT: v_readlane_b32 s12, v33, 53
+; VI-NEXT: v_mov_b32_e32 v3, s38
+; VI-NEXT: v_readlane_b32 s12, v33, 44
; VI-NEXT: v_perm_b32 v3, s12, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x58, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 52
+; VI-NEXT: v_readlane_b32 s12, v33, 43
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_mov_b32_e32 v2, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 50
+; VI-NEXT: v_readlane_b32 s12, v33, 41
; VI-NEXT: v_mov_b32_e32 v3, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 51
+; VI-NEXT: v_readlane_b32 s12, v33, 42
; VI-NEXT: v_perm_b32 v3, s12, v3, v1
; VI-NEXT: v_perm_b32 v2, s13, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x5c, v0
-; VI-NEXT: v_readlane_b32 s12, v33, 49
+; VI-NEXT: v_readlane_b32 s12, v33, 40
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; VI-NEXT: v_mov_b32_e32 v2, s12
+; VI-NEXT: v_readlane_b32 s12, v33, 22
+; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_perm_b32 v3, s90, v3, v1
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s60
-; VI-NEXT: v_readlane_b32 s10, v33, 48
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x60, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 47
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 45
-; VI-NEXT: v_mov_b32_e32 v3, s10
-; VI-NEXT: v_readlane_b32 s10, v33, 46
-; VI-NEXT: v_perm_b32 v3, s10, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s85
+; VI-NEXT: v_mov_b32_e32 v2, s91
+; VI-NEXT: v_perm_b32 v3, s88, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x64, v0
-; VI-NEXT: v_readlane_b32 s10, v33, 44
+; VI-NEXT: v_readlane_b32 s10, v33, 24
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s10
+; VI-NEXT: v_mov_b32_e32 v3, s10
+; VI-NEXT: v_mov_b32_e32 v2, s78
+; VI-NEXT: v_perm_b32 v3, s76, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s58
-; VI-NEXT: v_readlane_b32 s8, v33, 43
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x68, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 42
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 40
-; VI-NEXT: v_mov_b32_e32 v3, s8
-; VI-NEXT: v_readlane_b32 s8, v33, 41
-; VI-NEXT: v_perm_b32 v3, s8, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s61
+; VI-NEXT: v_mov_b32_e32 v2, s74
+; VI-NEXT: v_perm_b32 v3, s72, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x6c, v0
-; VI-NEXT: v_readlane_b32 s8, v33, 39
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s34
+; VI-NEXT: v_mov_b32_e32 v2, s60
+; VI-NEXT: v_perm_b32 v3, s67, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s56
-; VI-NEXT: v_readlane_b32 s6, v33, 38
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x70, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 37
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 35
-; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_readlane_b32 s6, v33, 36
-; VI-NEXT: v_perm_b32 v3, s6, v3, v1
+; VI-NEXT: v_mov_b32_e32 v3, s47
+; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_perm_b32 v3, s54, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x74, v0
-; VI-NEXT: v_readlane_b32 s6, v33, 34
+; VI-NEXT: v_readlane_b32 s6, v33, 26
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_mov_b32_e32 v2, s46
+; VI-NEXT: v_perm_b32 v3, s53, v3, v1
; VI-NEXT: v_perm_b32 v2, s4, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s46
-; VI-NEXT: v_readlane_b32 s4, v33, 33
-; VI-NEXT: v_perm_b32 v3, s4, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x78, v0
-; VI-NEXT: v_readlane_b32 s4, v33, 32
; VI-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 30
-; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: v_readlane_b32 s4, v33, 31
+; VI-NEXT: v_mov_b32_e32 v2, s64
+; VI-NEXT: v_mov_b32_e32 v3, s48
; VI-NEXT: v_perm_b32 v2, s5, v2, v1
-; VI-NEXT: v_perm_b32 v1, s4, v3, v1
+; VI-NEXT: v_perm_b32 v1, s49, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7c, v0
; VI-NEXT: v_readlane_b32 s30, v32, 30
+; VI-NEXT: v_readlane_b32 s17, v33, 21
+; VI-NEXT: v_readlane_b32 s13, v33, 23
+; VI-NEXT: v_readlane_b32 s11, v33, 25
+; VI-NEXT: v_readlane_b32 s7, v33, 27
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -204718,162 +205697,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB99_4:
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr68
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr81
-; VI-NEXT: ; implicit-def: $sgpr82
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr86
-; VI-NEXT: ; implicit-def: $sgpr87
-; VI-NEXT: ; implicit-def: $sgpr71
-; VI-NEXT: ; implicit-def: $sgpr70
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr85
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr80
-; VI-NEXT: ; implicit-def: $sgpr84
-; VI-NEXT: ; implicit-def: $sgpr83
-; VI-NEXT: ; implicit-def: $sgpr69
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; kill: killed $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB99_2
;
; GFX9-LABEL: bitcast_v64i16_to_v128i8_scalar:
; GFX9: ; %bb.0:
@@ -204953,7 +205776,7 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
-; GFX9-NEXT: s_cbranch_scc0 .LBB99_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s5, 24
; GFX9-NEXT: v_writelane_b32 v62, s46, 49
@@ -205098,11 +205921,165 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_lshr_b64 s[90:91], s[10:11], 24
; GFX9-NEXT: s_lshr_b64 s[92:93], s[12:13], 24
; GFX9-NEXT: s_lshr_b64 s[94:95], s[14:15], 24
-; GFX9-NEXT: s_lshr_b64 s[30:31], s[40:41], 24
-; GFX9-NEXT: s_lshr_b64 s[34:35], s[42:43], 24
-; GFX9-NEXT: s_lshr_b64 s[36:37], s[44:45], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB99_4
-; GFX9-NEXT: .LBB99_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b64 vcc, s[40:41], 24
+; GFX9-NEXT: s_lshr_b64 s[30:31], s[42:43], 24
+; GFX9-NEXT: s_lshr_b64 s[34:35], s[44:45], 24
+; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: s_branch .LBB99_3
+; GFX9-NEXT: .LBB99_2:
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: s_mov_b64 s[36:37], -1
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr81
+; GFX9-NEXT: ; implicit-def: $sgpr71
+; GFX9-NEXT: ; implicit-def: $sgpr80
+; GFX9-NEXT: ; implicit-def: $sgpr70
+; GFX9-NEXT: ; implicit-def: $sgpr69
+; GFX9-NEXT: ; implicit-def: $sgpr68
+; GFX9-NEXT: ; implicit-def: $sgpr66
+; GFX9-NEXT: ; implicit-def: $sgpr67
+; GFX9-NEXT: ; implicit-def: $sgpr65
+; GFX9-NEXT: ; implicit-def: $sgpr64
+; GFX9-NEXT: ; implicit-def: $sgpr55
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr54
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr99
+; GFX9-NEXT: ; implicit-def: $sgpr97
+; GFX9-NEXT: ; implicit-def: $sgpr98
+; GFX9-NEXT: ; implicit-def: $sgpr96
+; GFX9-NEXT: ; implicit-def: $sgpr87
+; GFX9-NEXT: ; implicit-def: $sgpr86
+; GFX9-NEXT: ; implicit-def: $sgpr84
+; GFX9-NEXT: ; implicit-def: $sgpr85
+; GFX9-NEXT: ; implicit-def: $sgpr83
+; GFX9-NEXT: ; implicit-def: $sgpr82
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; kill: killed $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB99_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GFX9-NEXT: s_cselect_b32 s47, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s47, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v26, s5, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v25, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[25:26]
@@ -205327,156 +206304,8 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 8, v19
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; GFX9-NEXT: s_branch .LBB99_5
-; GFX9-NEXT: .LBB99_3:
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr81
-; GFX9-NEXT: ; implicit-def: $sgpr71
-; GFX9-NEXT: ; implicit-def: $sgpr80
-; GFX9-NEXT: ; implicit-def: $sgpr70
-; GFX9-NEXT: ; implicit-def: $sgpr69
-; GFX9-NEXT: ; implicit-def: $sgpr68
-; GFX9-NEXT: ; implicit-def: $sgpr66
-; GFX9-NEXT: ; implicit-def: $sgpr67
-; GFX9-NEXT: ; implicit-def: $sgpr65
-; GFX9-NEXT: ; implicit-def: $sgpr64
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr99
-; GFX9-NEXT: ; implicit-def: $sgpr97
-; GFX9-NEXT: ; implicit-def: $sgpr98
-; GFX9-NEXT: ; implicit-def: $sgpr96
-; GFX9-NEXT: ; implicit-def: $sgpr87
-; GFX9-NEXT: ; implicit-def: $sgpr86
-; GFX9-NEXT: ; implicit-def: $sgpr84
-; GFX9-NEXT: ; implicit-def: $sgpr85
-; GFX9-NEXT: ; implicit-def: $sgpr83
-; GFX9-NEXT: ; implicit-def: $sgpr82
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; kill: killed $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB99_2
-; GFX9-NEXT: .LBB99_4:
+; GFX9-NEXT: s_branch .LBB99_6
+; GFX9-NEXT: .LBB99_5:
; GFX9-NEXT: v_mov_b32_e32 v15, s81
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s71
@@ -205742,19 +206571,19 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s30
+; GFX9-NEXT: v_mov_b32_e32 v22, vcc_lo
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s34
+; GFX9-NEXT: v_mov_b32_e32 v22, s30
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v22, s36
+; GFX9-NEXT: v_mov_b32_e32 v22, s34
; GFX9-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; GFX9-NEXT: .LBB99_5: ; %end
+; GFX9-NEXT: .LBB99_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
@@ -206211,74 +207040,74 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s41, v2
; GFX11-NEXT: v_readfirstlane_b32 s40, v1
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
+; GFX11-NEXT: s_mov_b32 s100, 0
; GFX11-NEXT: ; implicit-def: $vgpr76 : SGPR spill to VGPR lane
; GFX11-NEXT: ; implicit-def: $vgpr77 : SGPR spill to VGPR lane
-; GFX11-NEXT: s_cbranch_scc0 .LBB99_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s27, 24
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s5, 24
+; GFX11-NEXT: s_lshr_b32 s30, s5, 24
; GFX11-NEXT: v_writelane_b32 v77, s42, 7
; GFX11-NEXT: s_lshr_b32 s42, s27, 16
-; GFX11-NEXT: s_lshr_b32 s34, s5, 16
-; GFX11-NEXT: s_lshr_b32 s36, s5, 8
-; GFX11-NEXT: s_lshr_b32 s35, s4, 16
+; GFX11-NEXT: s_lshr_b32 s31, s5, 16
+; GFX11-NEXT: s_lshr_b32 s35, s5, 8
+; GFX11-NEXT: s_lshr_b32 s34, s4, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 6
; GFX11-NEXT: s_lshr_b32 s42, s27, 8
-; GFX11-NEXT: s_lshr_b32 s37, s4, 8
-; GFX11-NEXT: s_lshr_b32 s38, s7, 24
-; GFX11-NEXT: s_lshr_b32 s39, s7, 16
+; GFX11-NEXT: s_lshr_b32 s36, s4, 8
+; GFX11-NEXT: s_lshr_b32 s37, s7, 24
+; GFX11-NEXT: s_lshr_b32 s38, s7, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 5
; GFX11-NEXT: s_lshr_b32 s42, s26, 16
-; GFX11-NEXT: s_lshr_b32 s49, s7, 8
-; GFX11-NEXT: s_lshr_b32 s48, s6, 16
-; GFX11-NEXT: s_lshr_b32 s50, s6, 8
+; GFX11-NEXT: s_lshr_b32 s48, s7, 8
+; GFX11-NEXT: s_lshr_b32 s39, s6, 16
+; GFX11-NEXT: s_lshr_b32 s49, s6, 8
; GFX11-NEXT: v_writelane_b32 v77, s42, 4
; GFX11-NEXT: s_lshr_b32 s42, s26, 8
-; GFX11-NEXT: s_lshr_b32 s51, s9, 24
-; GFX11-NEXT: s_lshr_b32 s52, s9, 16
-; GFX11-NEXT: s_lshr_b32 s54, s9, 8
+; GFX11-NEXT: s_lshr_b32 s50, s9, 24
+; GFX11-NEXT: s_lshr_b32 s51, s9, 16
+; GFX11-NEXT: s_lshr_b32 s53, s9, 8
; GFX11-NEXT: v_writelane_b32 v77, s42, 3
; GFX11-NEXT: s_lshr_b32 s42, s25, 24
-; GFX11-NEXT: s_lshr_b32 s53, s8, 16
-; GFX11-NEXT: s_lshr_b32 s55, s8, 8
-; GFX11-NEXT: s_lshr_b32 s64, s11, 24
+; GFX11-NEXT: s_lshr_b32 s52, s8, 16
+; GFX11-NEXT: s_lshr_b32 s54, s8, 8
+; GFX11-NEXT: s_lshr_b32 s55, s11, 24
; GFX11-NEXT: v_writelane_b32 v77, s42, 2
; GFX11-NEXT: s_lshr_b32 s42, s25, 16
-; GFX11-NEXT: s_lshr_b32 s65, s11, 16
-; GFX11-NEXT: s_lshr_b32 s67, s11, 8
-; GFX11-NEXT: s_lshr_b32 s66, s10, 16
+; GFX11-NEXT: s_lshr_b32 s64, s11, 16
+; GFX11-NEXT: s_lshr_b32 s66, s11, 8
+; GFX11-NEXT: s_lshr_b32 s65, s10, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 1
; GFX11-NEXT: s_lshr_b32 s42, s25, 8
-; GFX11-NEXT: s_lshr_b32 s68, s10, 8
-; GFX11-NEXT: s_lshr_b32 s69, s13, 24
-; GFX11-NEXT: s_lshr_b32 s70, s13, 16
+; GFX11-NEXT: s_lshr_b32 s67, s10, 8
+; GFX11-NEXT: s_lshr_b32 s68, s13, 24
+; GFX11-NEXT: s_lshr_b32 s69, s13, 16
; GFX11-NEXT: v_writelane_b32 v77, s42, 0
; GFX11-NEXT: s_lshr_b32 s42, s24, 16
-; GFX11-NEXT: s_lshr_b32 s80, s13, 8
+; GFX11-NEXT: s_lshr_b32 s71, s13, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 31
; GFX11-NEXT: s_lshr_b32 s42, s24, 8
-; GFX11-NEXT: s_lshr_b32 s71, s12, 16
-; GFX11-NEXT: s_lshr_b32 s81, s12, 8
-; GFX11-NEXT: s_lshr_b32 s82, s15, 24
+; GFX11-NEXT: s_lshr_b32 s70, s12, 16
+; GFX11-NEXT: s_lshr_b32 s80, s12, 8
+; GFX11-NEXT: s_lshr_b32 s81, s15, 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 30
; GFX11-NEXT: s_lshr_b32 s42, s23, 24
-; GFX11-NEXT: s_lshr_b32 s83, s15, 16
-; GFX11-NEXT: s_lshr_b32 s85, s15, 8
-; GFX11-NEXT: s_lshr_b32 s84, s14, 16
+; GFX11-NEXT: s_lshr_b32 s82, s15, 16
+; GFX11-NEXT: s_lshr_b32 s84, s15, 8
+; GFX11-NEXT: s_lshr_b32 s83, s14, 16
; GFX11-NEXT: v_writelane_b32 v76, s42, 29
; GFX11-NEXT: s_lshr_b32 s42, s23, 16
-; GFX11-NEXT: s_lshr_b32 s86, s14, 8
-; GFX11-NEXT: s_lshr_b32 s87, s41, 24
-; GFX11-NEXT: s_lshr_b32 s96, s41, 16
+; GFX11-NEXT: s_lshr_b32 s85, s14, 8
+; GFX11-NEXT: s_lshr_b32 s86, s41, 24
+; GFX11-NEXT: s_lshr_b32 s87, s41, 16
; GFX11-NEXT: v_writelane_b32 v76, s42, 28
; GFX11-NEXT: s_lshr_b32 s42, s23, 8
-; GFX11-NEXT: s_lshr_b32 s98, s41, 8
-; GFX11-NEXT: s_lshr_b32 s97, s40, 16
-; GFX11-NEXT: s_lshr_b32 s99, s40, 8
+; GFX11-NEXT: s_lshr_b32 s97, s41, 8
+; GFX11-NEXT: s_lshr_b32 s96, s40, 16
+; GFX11-NEXT: s_lshr_b32 s98, s40, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 27
; GFX11-NEXT: s_lshr_b32 s42, s22, 16
-; GFX11-NEXT: s_lshr_b32 s100, s29, 24
+; GFX11-NEXT: s_lshr_b32 s99, s29, 24
; GFX11-NEXT: s_lshr_b32 s101, s29, 16
; GFX11-NEXT: s_lshr_b32 s103, s29, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 26
@@ -206309,7 +207138,7 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_writelane_b32 v76, s42, 21
; GFX11-NEXT: s_lshr_b32 s42, s20, 8
; GFX11-NEXT: s_lshr_b64 s[94:95], s[40:41], 24
-; GFX11-NEXT: s_lshr_b64 s[30:31], s[28:29], 24
+; GFX11-NEXT: s_lshr_b64 vcc, s[28:29], 24
; GFX11-NEXT: v_writelane_b32 v76, s42, 20
; GFX11-NEXT: s_lshr_b32 s42, s19, 24
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -206362,141 +207191,11 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s42, s0, 8
; GFX11-NEXT: v_writelane_b32 v76, s42, 0
; GFX11-NEXT: s_lshr_b64 s[42:43], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB99_4
-; GFX11-NEXT: .LBB99_2: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v28, s19, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v18, s27, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v17, s26, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v27, s18, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v30, s17, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v29, s16, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v20, s25, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v19, s24, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v14, s41, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v13, s40, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v10, s13, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v9, s12, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v6, s9, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v5, s8, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v34, s3, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v33, s2, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v22, s23, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v21, s22, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v16, s29, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v15, s28, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v12, s15, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v11, s14, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v8, s11, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v7, s10, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v4, s7, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v3, s6, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v2, s5, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v1, s4, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v36, s1, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v35, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v24, s21, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v23, s20, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
-; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[27:28]
-; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[19:20]
-; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[29:30]
-; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[21:22]
-; GFX11-NEXT: v_lshrrev_b64 v[69:70], 24, v[33:34]
-; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[5:6]
-; GFX11-NEXT: v_lshrrev_b64 v[48:49], 24, v[9:10]
-; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[13:14]
-; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[23:24]
-; GFX11-NEXT: v_lshrrev_b64 v[70:71], 24, v[35:36]
-; GFX11-NEXT: v_lshrrev_b64 v[25:26], 24, v[1:2]
-; GFX11-NEXT: v_lshrrev_b64 v[31:32], 24, v[3:4]
-; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[7:8]
-; GFX11-NEXT: v_lshrrev_b64 v[49:50], 24, v[11:12]
-; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[15:16]
-; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v147, 16, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v146, 8, v18
-; GFX11-NEXT: v_lshrrev_b32_e32 v160, 16, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v150, 8, v17
-; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v20
-; GFX11-NEXT: v_lshrrev_b32_e32 v163, 16, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v151, 8, v19
-; GFX11-NEXT: v_lshrrev_b32_e32 v166, 24, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v165, 16, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v164, 8, v22
-; GFX11-NEXT: v_lshrrev_b32_e32 v178, 16, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v176, 8, v21
-; GFX11-NEXT: v_lshrrev_b32_e32 v180, 24, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v24
-; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v177, 8, v23
-; GFX11-NEXT: v_lshrrev_b32_e32 v42, 24, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v41, 16, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v28
-; GFX11-NEXT: v_lshrrev_b32_e32 v43, 16, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v182, 8, v27
-; GFX11-NEXT: v_lshrrev_b32_e32 v46, 24, v30
-; GFX11-NEXT: v_lshrrev_b32_e32 v44, 16, v30
-; GFX11-NEXT: v_lshrrev_b32_e32 v40, 8, v30
-; GFX11-NEXT: v_lshrrev_b32_e32 v47, 16, v29
-; GFX11-NEXT: v_lshrrev_b32_e32 v45, 8, v29
-; GFX11-NEXT: v_lshrrev_b32_e32 v59, 24, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v58, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v60, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v56, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v72, 24, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v62, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v61, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v73, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v63, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v50, 24, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 8, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v32, 8, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v81, 24, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v55, 8, v4
-; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v82, 8, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v86, 24, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v83, 8, v6
-; GFX11-NEXT: v_lshrrev_b32_e32 v97, 16, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v5
-; GFX11-NEXT: v_lshrrev_b32_e32 v100, 24, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v99, 16, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v96, 8, v8
-; GFX11-NEXT: v_lshrrev_b32_e32 v101, 16, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v98, 8, v7
-; GFX11-NEXT: v_lshrrev_b32_e32 v112, 24, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v102, 8, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v115, 16, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v113, 8, v9
-; GFX11-NEXT: v_lshrrev_b32_e32 v118, 24, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v117, 16, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v114, 8, v12
-; GFX11-NEXT: v_lshrrev_b32_e32 v119, 16, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v116, 8, v11
-; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v129, 16, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-NEXT: v_lshrrev_b32_e32 v133, 16, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v131, 8, v13
-; GFX11-NEXT: v_lshrrev_b32_e32 v144, 24, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v135, 16, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v16
-; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v15
-; GFX11-NEXT: v_lshrrev_b32_e32 v134, 8, v15
-; GFX11-NEXT: s_branch .LBB99_5
-; GFX11-NEXT: .LBB99_3:
+; GFX11-NEXT: s_branch .LBB99_3
+; GFX11-NEXT: .LBB99_2:
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
+; GFX11-NEXT: s_mov_b32 s100, -1
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr42
@@ -206518,44 +207217,44 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: ; implicit-def: $sgpr102
; GFX11-NEXT: ; implicit-def: $sgpr103
; GFX11-NEXT: ; implicit-def: $sgpr101
-; GFX11-NEXT: ; implicit-def: $sgpr100
; GFX11-NEXT: ; implicit-def: $sgpr99
-; GFX11-NEXT: ; implicit-def: $sgpr97
; GFX11-NEXT: ; implicit-def: $sgpr98
; GFX11-NEXT: ; implicit-def: $sgpr96
+; GFX11-NEXT: ; implicit-def: $sgpr97
; GFX11-NEXT: ; implicit-def: $sgpr87
; GFX11-NEXT: ; implicit-def: $sgpr86
-; GFX11-NEXT: ; implicit-def: $sgpr84
; GFX11-NEXT: ; implicit-def: $sgpr85
; GFX11-NEXT: ; implicit-def: $sgpr83
+; GFX11-NEXT: ; implicit-def: $sgpr84
; GFX11-NEXT: ; implicit-def: $sgpr82
; GFX11-NEXT: ; implicit-def: $sgpr81
-; GFX11-NEXT: ; implicit-def: $sgpr71
; GFX11-NEXT: ; implicit-def: $sgpr80
; GFX11-NEXT: ; implicit-def: $sgpr70
+; GFX11-NEXT: ; implicit-def: $sgpr71
; GFX11-NEXT: ; implicit-def: $sgpr69
; GFX11-NEXT: ; implicit-def: $sgpr68
-; GFX11-NEXT: ; implicit-def: $sgpr66
; GFX11-NEXT: ; implicit-def: $sgpr67
; GFX11-NEXT: ; implicit-def: $sgpr65
+; GFX11-NEXT: ; implicit-def: $sgpr66
; GFX11-NEXT: ; implicit-def: $sgpr64
; GFX11-NEXT: ; implicit-def: $sgpr55
-; GFX11-NEXT: ; implicit-def: $sgpr53
; GFX11-NEXT: ; implicit-def: $sgpr54
; GFX11-NEXT: ; implicit-def: $sgpr52
+; GFX11-NEXT: ; implicit-def: $sgpr53
; GFX11-NEXT: ; implicit-def: $sgpr51
; GFX11-NEXT: ; implicit-def: $sgpr50
-; GFX11-NEXT: ; implicit-def: $sgpr48
; GFX11-NEXT: ; implicit-def: $sgpr49
; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr48
; GFX11-NEXT: ; implicit-def: $sgpr38
; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr35
; GFX11-NEXT: ; implicit-def: $sgpr36
; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr31
; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $vcc_lo
; GFX11-NEXT: ; implicit-def: $sgpr94
; GFX11-NEXT: ; implicit-def: $sgpr92
; GFX11-NEXT: ; implicit-def: $sgpr90
@@ -206631,8 +207330,143 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: ; kill: killed $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; kill: killed $sgpr43
-; GFX11-NEXT: s_branch .LBB99_2
-; GFX11-NEXT: .LBB99_4:
+; GFX11-NEXT: .LBB99_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s43, s100, exec_lo
+; GFX11-NEXT: s_cselect_b32 s43, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s43, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_pk_add_u16 v28, s19, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v18, s27, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v17, s26, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v27, s18, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v30, s17, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v29, s16, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v20, s25, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v19, s24, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v14, s41, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v13, s40, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v10, s13, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v9, s12, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v6, s9, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v5, s8, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v34, s3, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v33, s2, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v22, s23, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v21, s22, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v16, s29, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v15, s28, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v12, s15, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v11, s14, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v8, s11, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v7, s10, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v4, s7, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v3, s6, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v2, s5, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v1, s4, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v36, s1, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v35, s0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v24, s21, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_add_u16 v23, s20, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
+; GFX11-NEXT: v_lshrrev_b64 v[64:65], 24, v[27:28]
+; GFX11-NEXT: v_lshrrev_b64 v[52:53], 24, v[19:20]
+; GFX11-NEXT: v_lshrrev_b64 v[65:66], 24, v[29:30]
+; GFX11-NEXT: v_lshrrev_b64 v[53:54], 24, v[21:22]
+; GFX11-NEXT: v_lshrrev_b64 v[69:70], 24, v[33:34]
+; GFX11-NEXT: v_lshrrev_b64 v[37:38], 24, v[5:6]
+; GFX11-NEXT: v_lshrrev_b64 v[48:49], 24, v[9:10]
+; GFX11-NEXT: v_lshrrev_b64 v[66:67], 24, v[13:14]
+; GFX11-NEXT: v_lshrrev_b64 v[54:55], 24, v[23:24]
+; GFX11-NEXT: v_lshrrev_b64 v[70:71], 24, v[35:36]
+; GFX11-NEXT: v_lshrrev_b64 v[25:26], 24, v[1:2]
+; GFX11-NEXT: v_lshrrev_b64 v[31:32], 24, v[3:4]
+; GFX11-NEXT: v_lshrrev_b64 v[38:39], 24, v[7:8]
+; GFX11-NEXT: v_lshrrev_b64 v[49:50], 24, v[11:12]
+; GFX11-NEXT: v_lshrrev_b64 v[67:68], 24, v[15:16]
+; GFX11-NEXT: v_lshrrev_b32_e32 v148, 24, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v147, 16, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v146, 8, v18
+; GFX11-NEXT: v_lshrrev_b32_e32 v160, 16, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v150, 8, v17
+; GFX11-NEXT: v_lshrrev_b32_e32 v162, 24, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v161, 16, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v149, 8, v20
+; GFX11-NEXT: v_lshrrev_b32_e32 v163, 16, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v151, 8, v19
+; GFX11-NEXT: v_lshrrev_b32_e32 v166, 24, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v165, 16, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v164, 8, v22
+; GFX11-NEXT: v_lshrrev_b32_e32 v178, 16, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v176, 8, v21
+; GFX11-NEXT: v_lshrrev_b32_e32 v180, 24, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v179, 16, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v167, 8, v24
+; GFX11-NEXT: v_lshrrev_b32_e32 v181, 16, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v177, 8, v23
+; GFX11-NEXT: v_lshrrev_b32_e32 v42, 24, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v41, 16, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v183, 8, v28
+; GFX11-NEXT: v_lshrrev_b32_e32 v43, 16, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v182, 8, v27
+; GFX11-NEXT: v_lshrrev_b32_e32 v46, 24, v30
+; GFX11-NEXT: v_lshrrev_b32_e32 v44, 16, v30
+; GFX11-NEXT: v_lshrrev_b32_e32 v40, 8, v30
+; GFX11-NEXT: v_lshrrev_b32_e32 v47, 16, v29
+; GFX11-NEXT: v_lshrrev_b32_e32 v45, 8, v29
+; GFX11-NEXT: v_lshrrev_b32_e32 v59, 24, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v58, 16, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v57, 8, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v60, 16, v33
+; GFX11-NEXT: v_lshrrev_b32_e32 v56, 8, v33
+; GFX11-NEXT: v_lshrrev_b32_e32 v72, 24, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v62, 16, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v61, 8, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v73, 16, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v63, 8, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v50, 24, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v26, 8, v2
+; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v32, 8, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v81, 24, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v55, 8, v4
+; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v82, 8, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v86, 24, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v83, 8, v6
+; GFX11-NEXT: v_lshrrev_b32_e32 v97, 16, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v87, 8, v5
+; GFX11-NEXT: v_lshrrev_b32_e32 v100, 24, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v99, 16, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v96, 8, v8
+; GFX11-NEXT: v_lshrrev_b32_e32 v101, 16, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v98, 8, v7
+; GFX11-NEXT: v_lshrrev_b32_e32 v112, 24, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v103, 16, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v102, 8, v10
+; GFX11-NEXT: v_lshrrev_b32_e32 v115, 16, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v113, 8, v9
+; GFX11-NEXT: v_lshrrev_b32_e32 v118, 24, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v117, 16, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v114, 8, v12
+; GFX11-NEXT: v_lshrrev_b32_e32 v119, 16, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v116, 8, v11
+; GFX11-NEXT: v_lshrrev_b32_e32 v130, 24, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v129, 16, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v128, 8, v14
+; GFX11-NEXT: v_lshrrev_b32_e32 v133, 16, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v131, 8, v13
+; GFX11-NEXT: v_lshrrev_b32_e32 v144, 24, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v135, 16, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v132, 8, v16
+; GFX11-NEXT: v_lshrrev_b32_e32 v145, 16, v15
+; GFX11-NEXT: v_lshrrev_b32_e32 v134, 8, v15
+; GFX11-NEXT: s_branch .LBB99_6
+; GFX11-NEXT: .LBB99_5:
; GFX11-NEXT: v_dual_mov_b32 v35, s0 :: v_dual_mov_b32 v36, s1
; GFX11-NEXT: v_readlane_b32 s0, v76, 0
; GFX11-NEXT: v_dual_mov_b32 v15, s28 :: v_dual_mov_b32 v16, s29
@@ -206672,47 +207506,47 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_mov_b32_e32 v57, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 8
; GFX11-NEXT: v_dual_mov_b32 v132, s103 :: v_dual_mov_b32 v135, s101
-; GFX11-NEXT: v_dual_mov_b32 v144, s100 :: v_dual_mov_b32 v131, s99
+; GFX11-NEXT: v_dual_mov_b32 v144, s99 :: v_dual_mov_b32 v131, s98
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v58, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 9
-; GFX11-NEXT: v_dual_mov_b32 v133, s97 :: v_dual_mov_b32 v128, s98
-; GFX11-NEXT: v_dual_mov_b32 v129, s96 :: v_dual_mov_b32 v130, s87
+; GFX11-NEXT: v_dual_mov_b32 v133, s96 :: v_dual_mov_b32 v128, s97
+; GFX11-NEXT: v_dual_mov_b32 v129, s87 :: v_dual_mov_b32 v130, s86
; GFX11-NEXT: v_mov_b32_e32 v59, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 10
-; GFX11-NEXT: v_dual_mov_b32 v116, s86 :: v_dual_mov_b32 v119, s84
-; GFX11-NEXT: v_dual_mov_b32 v114, s85 :: v_dual_mov_b32 v117, s83
+; GFX11-NEXT: v_dual_mov_b32 v116, s85 :: v_dual_mov_b32 v119, s83
+; GFX11-NEXT: v_dual_mov_b32 v114, s84 :: v_dual_mov_b32 v117, s82
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v45, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 11
-; GFX11-NEXT: v_dual_mov_b32 v118, s82 :: v_dual_mov_b32 v113, s81
-; GFX11-NEXT: v_dual_mov_b32 v115, s71 :: v_dual_mov_b32 v102, s80
+; GFX11-NEXT: v_dual_mov_b32 v118, s81 :: v_dual_mov_b32 v113, s80
+; GFX11-NEXT: v_dual_mov_b32 v115, s70 :: v_dual_mov_b32 v102, s71
; GFX11-NEXT: v_mov_b32_e32 v47, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 12
-; GFX11-NEXT: v_dual_mov_b32 v103, s70 :: v_dual_mov_b32 v112, s69
-; GFX11-NEXT: v_dual_mov_b32 v98, s68 :: v_dual_mov_b32 v101, s66
+; GFX11-NEXT: v_dual_mov_b32 v103, s69 :: v_dual_mov_b32 v112, s68
+; GFX11-NEXT: v_dual_mov_b32 v98, s67 :: v_dual_mov_b32 v101, s65
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v40, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 13
-; GFX11-NEXT: v_dual_mov_b32 v96, s67 :: v_dual_mov_b32 v99, s65
-; GFX11-NEXT: v_dual_mov_b32 v100, s64 :: v_dual_mov_b32 v87, s55
+; GFX11-NEXT: v_dual_mov_b32 v96, s66 :: v_dual_mov_b32 v99, s64
+; GFX11-NEXT: v_dual_mov_b32 v100, s55 :: v_dual_mov_b32 v87, s54
; GFX11-NEXT: v_mov_b32_e32 v44, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 14
-; GFX11-NEXT: v_dual_mov_b32 v97, s53 :: v_dual_mov_b32 v86, s51
-; GFX11-NEXT: v_dual_mov_b32 v83, s54 :: v_dual_mov_b32 v82, s50
+; GFX11-NEXT: v_dual_mov_b32 v97, s52 :: v_dual_mov_b32 v86, s50
+; GFX11-NEXT: v_dual_mov_b32 v83, s53 :: v_dual_mov_b32 v82, s49
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v46, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 15
-; GFX11-NEXT: v_dual_mov_b32 v85, s52 :: v_dual_mov_b32 v84, s48
-; GFX11-NEXT: v_dual_mov_b32 v55, s49 :: v_dual_mov_b32 v80, s39
+; GFX11-NEXT: v_dual_mov_b32 v85, s51 :: v_dual_mov_b32 v84, s39
+; GFX11-NEXT: v_dual_mov_b32 v55, s48 :: v_dual_mov_b32 v80, s38
; GFX11-NEXT: v_mov_b32_e32 v182, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 16
-; GFX11-NEXT: v_dual_mov_b32 v81, s38 :: v_dual_mov_b32 v32, s37
-; GFX11-NEXT: v_dual_mov_b32 v71, s35 :: v_dual_mov_b32 v26, s36
+; GFX11-NEXT: v_dual_mov_b32 v81, s37 :: v_dual_mov_b32 v32, s36
+; GFX11-NEXT: v_dual_mov_b32 v71, s34 :: v_dual_mov_b32 v26, s35
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mov_b32_e32 v43, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 17
-; GFX11-NEXT: v_dual_mov_b32 v39, s34 :: v_dual_mov_b32 v50, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v39, s31 :: v_dual_mov_b32 v50, s30
; GFX11-NEXT: v_dual_mov_b32 v51, s72 :: v_dual_mov_b32 v52, s62
; GFX11-NEXT: v_mov_b32_e32 v183, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 18
@@ -206731,7 +207565,7 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_mov_b32_e32 v177, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 21
; GFX11-NEXT: v_mov_b32_e32 v49, s92
-; GFX11-NEXT: v_mov_b32_e32 v67, s30
+; GFX11-NEXT: v_mov_b32_e32 v67, vcc_lo
; GFX11-NEXT: v_mov_b32_e32 v181, s0
; GFX11-NEXT: v_readlane_b32 s0, v76, 22
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -206778,7 +207612,7 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX11-NEXT: v_mov_b32_e32 v147, s0
; GFX11-NEXT: v_readlane_b32 s0, v77, 7
; GFX11-NEXT: v_mov_b32_e32 v148, s0
-; GFX11-NEXT: .LBB99_5: ; %end
+; GFX11-NEXT: .LBB99_6: ; %end
; GFX11-NEXT: v_perm_b32 v68, v73, v70, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v70, v62, v72, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v69, v60, v69, 0xc0c0004
@@ -210716,7 +211550,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; SI-NEXT: s_cbranch_scc0 .LBB101_4
+; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_mov_b32_e32 v43, v44
; SI-NEXT: v_mov_b32_e32 v60, v2
@@ -210762,6 +211596,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_mov_b32_e32 v24, v6
; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v60
; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v38
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
@@ -211101,8 +211936,186 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_lshr_b64 v[10:11], v[6:7], 16
; SI-NEXT: v_mov_b32_e32 v11, v40
; SI-NEXT: v_lshr_b64 v[40:41], v[38:39], 16
-; SI-NEXT: s_cbranch_execnz .LBB101_3
-; SI-NEXT: .LBB101_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: v_mov_b32_e32 v11, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v48, v41
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: v_mov_b32_e32 v14, v61
+; SI-NEXT: v_mov_b32_e32 v19, v6
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_mov_b32_e32 v17, v16
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v60, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v9, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v17
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
@@ -211630,7 +212643,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_mov_b32_e32 v54, v43
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: .LBB101_3: ; %end
+; SI-NEXT: .LBB101_5: ; %end
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
@@ -211883,179 +212896,6 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB101_4:
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_mov_b32_e32 v11, v40
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v41
-; SI-NEXT: v_mov_b32_e32 v14, v61
-; SI-NEXT: v_mov_b32_e32 v19, v6
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr47
-; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr57
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr58
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v17, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v60, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v9, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v0
-; SI-NEXT: s_branch .LBB101_2
;
; VI-LABEL: bitcast_v64bf16_to_v64f16_scalar:
; VI: ; %bb.0:
@@ -212093,10 +212933,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB101_3
+; VI-NEXT: s_cbranch_scc0 .LBB101_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB101_4
-; VI-NEXT: .LBB101_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB101_3
+; VI-NEXT: .LBB101_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB101_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB101_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s48, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -212689,10 +213537,8 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB101_5
-; VI-NEXT: .LBB101_3:
-; VI-NEXT: s_branch .LBB101_2
-; VI-NEXT: .LBB101_4:
+; VI-NEXT: s_branch .LBB101_6
+; VI-NEXT: .LBB101_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -212725,7 +213571,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s49
; VI-NEXT: v_mov_b32_e32 v30, s50
; VI-NEXT: v_mov_b32_e32 v31, s51
-; VI-NEXT: .LBB101_5: ; %end
+; VI-NEXT: .LBB101_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v35, 8
; VI-NEXT: v_readlane_b32 s31, v35, 9
; VI-NEXT: v_readlane_b32 s51, v35, 7
@@ -212778,10 +213624,18 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB101_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB101_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB101_4
-; GFX9-NEXT: .LBB101_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB101_3
+; GFX9-NEXT: .LBB101_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB101_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB101_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: s_and_b32 s4, s50, 0xffff0000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -213392,10 +214246,8 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_and_b32_sdwa v16, v32, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; GFX9-NEXT: v_lshl_or_b32 v16, v32, 16, v16
-; GFX9-NEXT: s_branch .LBB101_5
-; GFX9-NEXT: .LBB101_3:
-; GFX9-NEXT: s_branch .LBB101_2
-; GFX9-NEXT: .LBB101_4:
+; GFX9-NEXT: s_branch .LBB101_6
+; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s17
@@ -213428,7 +214280,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s49
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
-; GFX9-NEXT: .LBB101_5: ; %end
+; GFX9-NEXT: .LBB101_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v36, 8
; GFX9-NEXT: v_readlane_b32 s31, v36, 9
; GFX9-NEXT: v_readlane_b32 s51, v36, 7
@@ -213482,11 +214334,16 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s13, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-TRUE16-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB101_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s36, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s36, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -214101,8 +214958,6 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v84
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v87.l
; GFX11-TRUE16-NEXT: s_branch .LBB101_5
-; GFX11-TRUE16-NEXT: .LBB101_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB101_2
; GFX11-TRUE16-NEXT: .LBB101_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -214172,11 +215027,16 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_mov_b32 s13, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-FAKE16-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB101_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s0, s36, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s36, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -214837,8 +215697,6 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v32, 16, v39
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v16, 16, v48
; GFX11-FAKE16-NEXT: s_branch .LBB101_5
-; GFX11-FAKE16-NEXT: .LBB101_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB101_2
; GFX11-FAKE16-NEXT: .LBB101_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -216283,12 +217141,10 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: s_lshr_b32 s10, s21, 16
; SI-NEXT: v_writelane_b32 v62, s9, 4
; SI-NEXT: s_lshr_b32 s11, s22, 16
-; SI-NEXT: v_writelane_b32 v62, s10, 5
-; SI-NEXT: s_lshr_b32 s12, s23, 16
-; SI-NEXT: v_readfirstlane_b32 s52, v17
-; SI-NEXT: v_readfirstlane_b32 s48, v16
-; SI-NEXT: v_readfirstlane_b32 s35, v15
-; SI-NEXT: v_readfirstlane_b32 s30, v14
+; SI-NEXT: v_readfirstlane_b32 s50, v17
+; SI-NEXT: v_readfirstlane_b32 s38, v16
+; SI-NEXT: v_readfirstlane_b32 s31, v15
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v14
; SI-NEXT: v_readfirstlane_b32 s94, v13
; SI-NEXT: v_readfirstlane_b32 s92, v12
; SI-NEXT: v_readfirstlane_b32 s90, v11
@@ -216303,17 +217159,18 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_readfirstlane_b32 s56, v2
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: v_readfirstlane_b32 s44, v0
-; SI-NEXT: v_writelane_b32 v62, s11, 6
+; SI-NEXT: v_writelane_b32 v62, s10, 5
; SI-NEXT: s_lshr_b32 s43, s29, 16
; SI-NEXT: s_lshr_b32 s42, s28, 16
; SI-NEXT: s_lshr_b32 s41, s27, 16
; SI-NEXT: s_lshr_b32 s40, s26, 16
; SI-NEXT: s_lshr_b32 s15, s25, 16
-; SI-NEXT: s_lshr_b32 s13, s24, 16
-; SI-NEXT: s_lshr_b32 s55, s52, 16
-; SI-NEXT: s_lshr_b32 s51, s48, 16
-; SI-NEXT: s_lshr_b32 s38, s35, 16
-; SI-NEXT: s_lshr_b32 s31, s30, 16
+; SI-NEXT: s_lshr_b32 s14, s24, 16
+; SI-NEXT: s_lshr_b32 s12, s23, 16
+; SI-NEXT: s_lshr_b32 s53, s50, 16
+; SI-NEXT: s_lshr_b32 s49, s38, 16
+; SI-NEXT: s_lshr_b32 s36, s31, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s91, s90, 16
@@ -216329,108 +217186,205 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: s_lshr_b32 s47, s46, 16
; SI-NEXT: s_lshr_b32 s45, s44, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v62, s12, 7
+; SI-NEXT: v_writelane_b32 v62, s11, 6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v62, s13, 8
-; SI-NEXT: s_cbranch_scc0 .LBB103_3
+; SI-NEXT: v_writelane_b32 v62, s12, 7
+; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s4, s16, 16
-; SI-NEXT: v_writelane_b32 v62, s4, 35
-; SI-NEXT: s_lshl_b32 s4, s5, 16
-; SI-NEXT: v_writelane_b32 v62, s4, 34
-; SI-NEXT: s_lshl_b32 s4, s17, 16
; SI-NEXT: v_writelane_b32 v62, s4, 33
-; SI-NEXT: s_lshl_b32 s4, s6, 16
+; SI-NEXT: s_lshl_b32 s4, s5, 16
; SI-NEXT: v_writelane_b32 v62, s4, 32
-; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: s_lshl_b32 s4, s17, 16
; SI-NEXT: v_writelane_b32 v62, s4, 31
-; SI-NEXT: s_lshl_b32 s4, s7, 16
+; SI-NEXT: s_lshl_b32 s4, s6, 16
; SI-NEXT: v_writelane_b32 v62, s4, 30
-; SI-NEXT: s_lshl_b32 s4, s19, 16
+; SI-NEXT: s_lshl_b32 s4, s18, 16
; SI-NEXT: v_writelane_b32 v62, s4, 29
-; SI-NEXT: s_lshl_b32 s4, s8, 16
+; SI-NEXT: s_lshl_b32 s4, s7, 16
; SI-NEXT: v_writelane_b32 v62, s4, 28
-; SI-NEXT: s_lshl_b32 s4, s20, 16
+; SI-NEXT: s_lshl_b32 s4, s19, 16
; SI-NEXT: v_writelane_b32 v62, s4, 27
-; SI-NEXT: s_lshl_b32 s4, s9, 16
+; SI-NEXT: s_lshl_b32 s4, s8, 16
; SI-NEXT: v_writelane_b32 v62, s4, 26
-; SI-NEXT: s_lshl_b32 s4, s21, 16
+; SI-NEXT: s_lshl_b32 s4, s20, 16
; SI-NEXT: v_writelane_b32 v62, s4, 25
-; SI-NEXT: s_lshl_b32 s4, s10, 16
+; SI-NEXT: s_lshl_b32 s4, s9, 16
; SI-NEXT: v_writelane_b32 v62, s4, 24
-; SI-NEXT: s_lshl_b32 s4, s22, 16
+; SI-NEXT: s_lshl_b32 s4, s21, 16
; SI-NEXT: v_writelane_b32 v62, s4, 23
-; SI-NEXT: s_lshl_b32 s4, s11, 16
+; SI-NEXT: s_lshl_b32 s4, s10, 16
; SI-NEXT: v_writelane_b32 v62, s4, 22
-; SI-NEXT: s_lshl_b32 s4, s23, 16
+; SI-NEXT: s_lshl_b32 s4, s22, 16
; SI-NEXT: v_writelane_b32 v62, s4, 21
-; SI-NEXT: s_lshl_b32 s4, s12, 16
+; SI-NEXT: s_lshl_b32 s4, s11, 16
; SI-NEXT: v_writelane_b32 v62, s4, 20
-; SI-NEXT: s_lshl_b32 s4, s24, 16
+; SI-NEXT: s_lshl_b32 s4, s23, 16
; SI-NEXT: v_writelane_b32 v62, s4, 19
-; SI-NEXT: s_lshl_b32 s4, s13, 16
+; SI-NEXT: s_lshl_b32 s4, s12, 16
; SI-NEXT: v_writelane_b32 v62, s4, 18
-; SI-NEXT: s_lshl_b32 s4, s25, 16
+; SI-NEXT: s_lshl_b32 s4, s24, 16
; SI-NEXT: v_writelane_b32 v62, s4, 17
-; SI-NEXT: s_lshl_b32 s4, s15, 16
+; SI-NEXT: s_lshl_b32 s4, s14, 16
; SI-NEXT: v_writelane_b32 v62, s4, 16
-; SI-NEXT: s_lshl_b32 s4, s26, 16
+; SI-NEXT: s_lshl_b32 s4, s25, 16
; SI-NEXT: v_writelane_b32 v62, s4, 15
-; SI-NEXT: s_lshl_b32 s4, s40, 16
+; SI-NEXT: s_lshl_b32 s4, s15, 16
; SI-NEXT: v_writelane_b32 v62, s4, 14
-; SI-NEXT: s_lshl_b32 s4, s27, 16
+; SI-NEXT: s_lshl_b32 s4, s26, 16
; SI-NEXT: v_writelane_b32 v62, s4, 13
-; SI-NEXT: s_lshl_b32 s4, s41, 16
+; SI-NEXT: s_lshl_b32 s4, s40, 16
; SI-NEXT: v_writelane_b32 v62, s4, 12
-; SI-NEXT: s_lshl_b32 s4, s28, 16
+; SI-NEXT: s_lshl_b32 s4, s27, 16
; SI-NEXT: v_writelane_b32 v62, s4, 11
-; SI-NEXT: s_lshl_b32 s4, s42, 16
+; SI-NEXT: s_lshl_b32 s4, s41, 16
; SI-NEXT: v_writelane_b32 v62, s4, 10
-; SI-NEXT: s_lshl_b32 s4, s29, 16
+; SI-NEXT: s_lshl_b32 s4, s28, 16
; SI-NEXT: v_writelane_b32 v62, s4, 9
-; SI-NEXT: s_lshl_b32 s99, s43, 16
-; SI-NEXT: s_lshl_b32 s34, s44, 16
-; SI-NEXT: s_lshl_b32 s36, s45, 16
-; SI-NEXT: s_lshl_b32 s37, s46, 16
-; SI-NEXT: s_lshl_b32 s39, s47, 16
-; SI-NEXT: s_lshl_b32 s49, s56, 16
-; SI-NEXT: s_lshl_b32 s50, s57, 16
-; SI-NEXT: s_lshl_b32 s53, s58, 16
-; SI-NEXT: s_lshl_b32 s54, s59, 16
-; SI-NEXT: s_lshl_b32 s64, s60, 16
-; SI-NEXT: s_lshl_b32 s65, s61, 16
-; SI-NEXT: s_lshl_b32 s66, s62, 16
-; SI-NEXT: s_lshl_b32 s67, s63, 16
-; SI-NEXT: s_lshl_b32 s6, s72, 16
-; SI-NEXT: s_lshl_b32 s68, s73, 16
-; SI-NEXT: s_lshl_b32 s7, s74, 16
-; SI-NEXT: s_lshl_b32 s69, s75, 16
-; SI-NEXT: s_lshl_b32 s70, s76, 16
-; SI-NEXT: s_lshl_b32 s71, s77, 16
-; SI-NEXT: s_lshl_b32 s8, s78, 16
-; SI-NEXT: s_lshl_b32 s80, s79, 16
-; SI-NEXT: s_lshl_b32 s81, s88, 16
-; SI-NEXT: s_lshl_b32 s9, s89, 16
-; SI-NEXT: s_lshl_b32 s82, s90, 16
-; SI-NEXT: s_lshl_b32 s83, s91, 16
-; SI-NEXT: s_lshl_b32 s10, s92, 16
-; SI-NEXT: s_lshl_b32 s84, s93, 16
-; SI-NEXT: s_lshl_b32 s85, s94, 16
-; SI-NEXT: s_lshl_b32 s11, s95, 16
-; SI-NEXT: s_lshl_b32 s86, s30, 16
-; SI-NEXT: s_lshl_b32 s87, s31, 16
-; SI-NEXT: s_lshl_b32 s12, s35, 16
-; SI-NEXT: s_lshl_b32 s96, s38, 16
-; SI-NEXT: s_lshl_b32 s13, s48, 16
-; SI-NEXT: s_lshl_b32 s97, s51, 16
-; SI-NEXT: s_lshl_b32 s98, s52, 16
-; SI-NEXT: s_lshl_b32 s14, s55, 16
-; SI-NEXT: s_cbranch_execnz .LBB103_4
-; SI-NEXT: .LBB103_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s55
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s52
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s51
-; SI-NEXT: v_readlane_b32 s4, v62, 8
+; SI-NEXT: s_lshl_b32 s4, s42, 16
+; SI-NEXT: v_writelane_b32 v62, s4, 8
+; SI-NEXT: s_lshl_b32 s96, s29, 16
+; SI-NEXT: s_lshl_b32 s97, s43, 16
+; SI-NEXT: s_lshl_b32 s98, s44, 16
+; SI-NEXT: s_lshl_b32 s99, s45, 16
+; SI-NEXT: s_lshl_b32 s30, s46, 16
+; SI-NEXT: s_lshl_b32 s34, s47, 16
+; SI-NEXT: s_lshl_b32 s35, s56, 16
+; SI-NEXT: s_lshl_b32 s37, s57, 16
+; SI-NEXT: s_lshl_b32 s39, s58, 16
+; SI-NEXT: s_lshl_b32 s48, s59, 16
+; SI-NEXT: s_lshl_b32 s51, s60, 16
+; SI-NEXT: s_lshl_b32 s52, s61, 16
+; SI-NEXT: s_lshl_b32 s54, s62, 16
+; SI-NEXT: s_lshl_b32 s55, s63, 16
+; SI-NEXT: s_lshl_b32 s64, s72, 16
+; SI-NEXT: s_lshl_b32 s65, s73, 16
+; SI-NEXT: s_lshl_b32 s6, s74, 16
+; SI-NEXT: s_lshl_b32 s66, s75, 16
+; SI-NEXT: s_lshl_b32 s7, s76, 16
+; SI-NEXT: s_lshl_b32 s67, s77, 16
+; SI-NEXT: s_lshl_b32 s68, s78, 16
+; SI-NEXT: s_lshl_b32 s69, s79, 16
+; SI-NEXT: s_lshl_b32 s8, s88, 16
+; SI-NEXT: s_lshl_b32 s70, s89, 16
+; SI-NEXT: s_lshl_b32 s71, s90, 16
+; SI-NEXT: s_lshl_b32 s9, s91, 16
+; SI-NEXT: s_lshl_b32 s80, s92, 16
+; SI-NEXT: s_lshl_b32 s81, s93, 16
+; SI-NEXT: s_lshl_b32 s10, s94, 16
+; SI-NEXT: s_lshl_b32 s82, s95, 16
+; SI-NEXT: s_lshl_b32 s83, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s11, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s84, s31, 16
+; SI-NEXT: s_lshl_b32 s85, s36, 16
+; SI-NEXT: s_lshl_b32 s12, s38, 16
+; SI-NEXT: s_lshl_b32 s86, s49, 16
+; SI-NEXT: s_lshl_b32 s13, s50, 16
+; SI-NEXT: s_lshl_b32 s87, s53, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB103_3
+; SI-NEXT: .LBB103_2:
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr97
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB103_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB103_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s53
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s50
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s49
+; SI-NEXT: v_readlane_b32 s4, v62, 7
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -216443,10 +217397,10 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s48
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s38
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s38
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s36
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -216455,25 +217409,25 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v1, vcc_hi
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s30
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: v_cvt_f32_f16_e32 v0, vcc_lo
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v1, s95
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v0, s94
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
@@ -216493,7 +217447,7 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v48, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v39, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s79
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -216525,30 +217479,28 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v54, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s74
-; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v41
-; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v48
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v40, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s72
; SI-NEXT: v_cvt_f16_f32_e32 v43, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s61
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v42, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v40, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s72
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v45, v1
+; SI-NEXT: v_lshlrev_b32_e32 v57, 16, v41
+; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
+; SI-NEXT: v_cvt_f16_f32_e32 v42, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s62
-; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v42
-; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v52
; SI-NEXT: v_lshlrev_b32_e32 v61, 16, v45
; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v53
-; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v42, 16, v52
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v44, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s60
+; SI-NEXT: v_lshlrev_b32_e32 v53, 16, v51
+; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v39
; SI-NEXT: v_cvt_f32_f16_e32 v1, s59
-; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v43
-; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v44
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v46, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s58
@@ -216596,24 +217548,23 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_cvt_f32_f16_e32 v1, s40
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v22, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v0, s26
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v21, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_cvt_f32_f16_e32 v1, s15
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v20, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s25
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v19, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s14
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v18, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
-; SI-NEXT: v_readlane_b32 s4, v62, 7
; SI-NEXT: v_cvt_f16_f32_e32 v17, v1
; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -216667,24 +217618,24 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v39, v1
+; SI-NEXT: v_cvt_f16_f32_e32 v48, v1
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v50
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v39
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v48
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v38
+; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v38
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v37
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v48, 16, v7
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_lshlrev_b32_e32 v50, 16, v36
@@ -216707,19 +217658,21 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshlrev_b32_e32 v36, 16, v34
; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v35
+; SI-NEXT: v_lshlrev_b32_e32 v58, 16, v44
+; SI-NEXT: v_lshlrev_b32_e32 v59, 16, v43
; SI-NEXT: v_lshlrev_b32_e32 v44, 16, v54
; SI-NEXT: v_lshlrev_b32_e32 v43, 16, v16
; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v8
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v51, 16, v7
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v7
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v7
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
@@ -216749,214 +217702,120 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: s_branch .LBB103_5
-; SI-NEXT: .LBB103_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB103_2
-; SI-NEXT: .LBB103_4:
+; SI-NEXT: s_branch .LBB103_6
+; SI-NEXT: .LBB103_5:
+; SI-NEXT: v_readlane_b32 s4, v62, 8
+; SI-NEXT: v_mov_b32_e32 v25, s4
; SI-NEXT: v_readlane_b32 s4, v62, 9
-; SI-NEXT: v_mov_b32_e32 v26, s4
+; SI-NEXT: v_mov_b32_e32 v24, s4
; SI-NEXT: v_readlane_b32 s4, v62, 10
-; SI-NEXT: v_mov_b32_e32 v25, s4
+; SI-NEXT: v_mov_b32_e32 v23, s4
; SI-NEXT: v_readlane_b32 s4, v62, 11
-; SI-NEXT: v_mov_b32_e32 v24, s4
+; SI-NEXT: v_mov_b32_e32 v22, s4
; SI-NEXT: v_readlane_b32 s4, v62, 12
-; SI-NEXT: v_mov_b32_e32 v23, s4
+; SI-NEXT: v_mov_b32_e32 v21, s4
; SI-NEXT: v_readlane_b32 s4, v62, 13
-; SI-NEXT: v_mov_b32_e32 v22, s4
+; SI-NEXT: v_mov_b32_e32 v20, s4
; SI-NEXT: v_readlane_b32 s4, v62, 14
-; SI-NEXT: v_mov_b32_e32 v21, s4
+; SI-NEXT: v_mov_b32_e32 v19, s4
; SI-NEXT: v_readlane_b32 s4, v62, 15
-; SI-NEXT: v_mov_b32_e32 v20, s4
+; SI-NEXT: v_mov_b32_e32 v18, s4
; SI-NEXT: v_readlane_b32 s4, v62, 16
-; SI-NEXT: v_mov_b32_e32 v19, s4
+; SI-NEXT: v_mov_b32_e32 v17, s4
; SI-NEXT: v_readlane_b32 s4, v62, 17
-; SI-NEXT: v_mov_b32_e32 v18, s4
+; SI-NEXT: v_mov_b32_e32 v50, s4
; SI-NEXT: v_readlane_b32 s4, v62, 18
-; SI-NEXT: v_mov_b32_e32 v17, s4
+; SI-NEXT: v_mov_b32_e32 v15, s4
; SI-NEXT: v_readlane_b32 s4, v62, 19
-; SI-NEXT: v_mov_b32_e32 v50, s4
+; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_readlane_b32 s4, v62, 20
-; SI-NEXT: v_mov_b32_e32 v15, s4
+; SI-NEXT: v_mov_b32_e32 v13, s4
; SI-NEXT: v_readlane_b32 s4, v62, 21
-; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v12, s4
; SI-NEXT: v_readlane_b32 s4, v62, 22
-; SI-NEXT: v_mov_b32_e32 v13, s4
+; SI-NEXT: v_mov_b32_e32 v11, s4
; SI-NEXT: v_readlane_b32 s4, v62, 23
-; SI-NEXT: v_mov_b32_e32 v12, s4
+; SI-NEXT: v_mov_b32_e32 v10, s4
; SI-NEXT: v_readlane_b32 s4, v62, 24
-; SI-NEXT: v_mov_b32_e32 v11, s4
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: v_readlane_b32 s4, v62, 25
-; SI-NEXT: v_mov_b32_e32 v10, s4
+; SI-NEXT: v_mov_b32_e32 v38, s4
; SI-NEXT: v_readlane_b32 s4, v62, 26
-; SI-NEXT: v_mov_b32_e32 v9, s4
+; SI-NEXT: v_mov_b32_e32 v0, s87
+; SI-NEXT: v_mov_b32_e32 v48, s4
; SI-NEXT: v_readlane_b32 s4, v62, 27
-; SI-NEXT: v_mov_b32_e32 v38, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 28
-; SI-NEXT: v_mov_b32_e32 v0, s14
-; SI-NEXT: v_mov_b32_e32 v39, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 29
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s98
+; SI-NEXT: v_mov_b32_e32 v0, s13
; SI-NEXT: v_mov_b32_e32 v6, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 30
+; SI-NEXT: v_readlane_b32 s4, v62, 28
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s97
+; SI-NEXT: v_mov_b32_e32 v0, s86
; SI-NEXT: v_mov_b32_e32 v5, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 31
+; SI-NEXT: v_readlane_b32 s4, v62, 29
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s13
+; SI-NEXT: v_mov_b32_e32 v0, s12
; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 32
+; SI-NEXT: v_readlane_b32 s4, v62, 30
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s96
+; SI-NEXT: v_mov_b32_e32 v0, s85
; SI-NEXT: v_mov_b32_e32 v3, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 33
+; SI-NEXT: v_readlane_b32 s4, v62, 31
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v0, s84
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 34
+; SI-NEXT: v_readlane_b32 s4, v62, 32
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s86
+; SI-NEXT: v_mov_b32_e32 v0, s11
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s4, v62, 35
-; SI-NEXT: v_mov_b32_e32 v49, s87
+; SI-NEXT: v_readlane_b32 s4, v62, 33
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v51, s11
-; SI-NEXT: v_mov_b32_e32 v48, s85
-; SI-NEXT: v_mov_b32_e32 v53, s84
-; SI-NEXT: v_mov_b32_e32 v52, s10
-; SI-NEXT: v_mov_b32_e32 v55, s83
-; SI-NEXT: v_mov_b32_e32 v54, s82
-; SI-NEXT: v_mov_b32_e32 v41, s9
-; SI-NEXT: v_mov_b32_e32 v40, s81
-; SI-NEXT: v_mov_b32_e32 v43, s80
-; SI-NEXT: v_mov_b32_e32 v42, s8
-; SI-NEXT: v_mov_b32_e32 v45, s71
-; SI-NEXT: v_mov_b32_e32 v44, s70
-; SI-NEXT: v_mov_b32_e32 v47, s69
-; SI-NEXT: v_mov_b32_e32 v46, s7
-; SI-NEXT: v_mov_b32_e32 v57, s68
-; SI-NEXT: v_mov_b32_e32 v56, s6
-; SI-NEXT: v_mov_b32_e32 v59, s67
-; SI-NEXT: v_mov_b32_e32 v58, s66
-; SI-NEXT: v_mov_b32_e32 v61, s65
-; SI-NEXT: v_mov_b32_e32 v60, s64
-; SI-NEXT: v_mov_b32_e32 v37, s54
-; SI-NEXT: v_mov_b32_e32 v36, s53
-; SI-NEXT: v_mov_b32_e32 v33, s50
-; SI-NEXT: v_mov_b32_e32 v32, s49
-; SI-NEXT: v_mov_b32_e32 v31, s39
-; SI-NEXT: v_mov_b32_e32 v30, s37
-; SI-NEXT: v_mov_b32_e32 v29, s36
-; SI-NEXT: v_mov_b32_e32 v28, s34
-; SI-NEXT: v_mov_b32_e32 v27, s99
+; SI-NEXT: v_mov_b32_e32 v39, s83
+; SI-NEXT: v_mov_b32_e32 v51, s82
+; SI-NEXT: v_mov_b32_e32 v49, s10
+; SI-NEXT: v_mov_b32_e32 v53, s81
+; SI-NEXT: v_mov_b32_e32 v52, s80
+; SI-NEXT: v_mov_b32_e32 v55, s9
+; SI-NEXT: v_mov_b32_e32 v54, s71
+; SI-NEXT: v_mov_b32_e32 v41, s70
+; SI-NEXT: v_mov_b32_e32 v40, s8
+; SI-NEXT: v_mov_b32_e32 v43, s69
+; SI-NEXT: v_mov_b32_e32 v42, s68
+; SI-NEXT: v_mov_b32_e32 v45, s67
+; SI-NEXT: v_mov_b32_e32 v44, s7
+; SI-NEXT: v_mov_b32_e32 v47, s66
+; SI-NEXT: v_mov_b32_e32 v46, s6
+; SI-NEXT: v_mov_b32_e32 v57, s65
+; SI-NEXT: v_mov_b32_e32 v56, s64
+; SI-NEXT: v_mov_b32_e32 v59, s55
+; SI-NEXT: v_mov_b32_e32 v58, s54
+; SI-NEXT: v_mov_b32_e32 v61, s52
+; SI-NEXT: v_mov_b32_e32 v60, s51
+; SI-NEXT: v_mov_b32_e32 v37, s48
+; SI-NEXT: v_mov_b32_e32 v36, s39
+; SI-NEXT: v_mov_b32_e32 v33, s37
+; SI-NEXT: v_mov_b32_e32 v32, s35
+; SI-NEXT: v_mov_b32_e32 v31, s34
+; SI-NEXT: v_mov_b32_e32 v30, s30
+; SI-NEXT: v_mov_b32_e32 v29, s99
+; SI-NEXT: v_mov_b32_e32 v28, s98
+; SI-NEXT: v_mov_b32_e32 v27, s97
+; SI-NEXT: v_mov_b32_e32 v26, s96
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: .LBB103_5: ; %end
+; SI-NEXT: .LBB103_6: ; %end
; SI-NEXT: v_mul_f32_e32 v34, 1.0, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v5
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v39
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v48
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v6
@@ -217024,11 +217883,10 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v51
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; SI-NEXT: v_mul_f32_e32 v0, 1.0, v49
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_mul_f32_e32 v9, 1.0, v50
-; SI-NEXT: v_mul_f32_e32 v50, 1.0, v48
+; SI-NEXT: v_mul_f32_e32 v50, 1.0, v49
+; SI-NEXT: v_mul_f32_e32 v48, 1.0, v39
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_mul_f32_e32 v7, 1.0, v4
; SI-NEXT: v_lshr_b64 v[33:34], v[34:35], 16
@@ -217090,7 +217948,8 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_mul_f32_e32 v48, 1.0, v0
+; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
@@ -217208,10 +218067,18 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB103_3
+; VI-NEXT: s_cbranch_scc0 .LBB103_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB103_4
-; VI-NEXT: .LBB103_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB103_3
+; VI-NEXT: .LBB103_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB103_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB103_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s50, 16
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_lshr_b32 s4, s30, 16
@@ -217373,10 +218240,8 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s36, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB103_5
-; VI-NEXT: .LBB103_3:
-; VI-NEXT: s_branch .LBB103_2
-; VI-NEXT: .LBB103_4:
+; VI-NEXT: s_branch .LBB103_6
+; VI-NEXT: .LBB103_5:
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v17, s37
@@ -217409,7 +218274,7 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB103_5: ; %end
+; VI-NEXT: .LBB103_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v33, 8
; VI-NEXT: v_readlane_b32 s31, v33, 9
; VI-NEXT: v_readlane_b32 s51, v33, 7
@@ -217462,10 +218327,18 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB103_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB103_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB103_4
-; GFX9-NEXT: .LBB103_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB103_3
+; GFX9-NEXT: .LBB103_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB103_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v16 op_sel_hi:[1,0]
@@ -217499,10 +218372,8 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; GFX9-NEXT: v_pk_add_f16 v18, s38, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s37, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s36, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB103_5
-; GFX9-NEXT: .LBB103_3:
-; GFX9-NEXT: s_branch .LBB103_2
-; GFX9-NEXT: .LBB103_4:
+; GFX9-NEXT: s_branch .LBB103_6
+; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -217535,7 +218406,7 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB103_5: ; %end
+; GFX9-NEXT: .LBB103_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -217589,11 +218460,16 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB103_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -217627,8 +218503,6 @@ define inreg <64 x bfloat> @bitcast_v64f16_to_v64bf16_scalar(<64 x half> inreg %
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s37 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s36 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB103_5
-; GFX11-NEXT: .LBB103_3:
-; GFX11-NEXT: s_branch .LBB103_2
; GFX11-NEXT: .LBB103_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -221560,8 +222434,10 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_cbranch_vccnz .LBB105_5
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
@@ -222263,10 +223139,18 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB105_3
+; VI-NEXT: s_cbranch_scc0 .LBB105_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB105_4
-; VI-NEXT: .LBB105_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB105_3
+; VI-NEXT: .LBB105_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB105_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB105_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v32, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s48, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v32
@@ -222859,10 +223743,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_cndmask_b32_e32 v31, v32, v34, vcc
; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
; VI-NEXT: v_lshrrev_b64 v[31:32], 16, v[31:32]
-; VI-NEXT: s_branch .LBB105_5
-; VI-NEXT: .LBB105_3:
-; VI-NEXT: s_branch .LBB105_2
-; VI-NEXT: .LBB105_4:
+; VI-NEXT: s_branch .LBB105_6
+; VI-NEXT: .LBB105_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -222895,7 +223777,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s49
; VI-NEXT: v_mov_b32_e32 v30, s50
; VI-NEXT: v_mov_b32_e32 v31, s51
-; VI-NEXT: .LBB105_5: ; %end
+; VI-NEXT: .LBB105_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v35, 8
; VI-NEXT: v_readlane_b32 s31, v35, 9
; VI-NEXT: v_readlane_b32 s51, v35, 7
@@ -222948,10 +223830,18 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB105_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB105_4
-; GFX9-NEXT: .LBB105_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB105_3
+; GFX9-NEXT: .LBB105_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB105_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x40c00000
; GFX9-NEXT: s_and_b32 s4, s50, 0xffff0000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v16
@@ -223530,10 +224420,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_cndmask_b32_e32 v16, v34, v35, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX9-NEXT: v_and_or_b32 v16, v33, v32, v16
-; GFX9-NEXT: s_branch .LBB105_5
-; GFX9-NEXT: .LBB105_3:
-; GFX9-NEXT: s_branch .LBB105_2
-; GFX9-NEXT: .LBB105_4:
+; GFX9-NEXT: s_branch .LBB105_6
+; GFX9-NEXT: .LBB105_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s17
@@ -223566,7 +224454,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s49
; GFX9-NEXT: v_mov_b32_e32 v30, s50
; GFX9-NEXT: v_mov_b32_e32 v31, s51
-; GFX9-NEXT: .LBB105_5: ; %end
+; GFX9-NEXT: .LBB105_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v36, 8
; GFX9-NEXT: v_readlane_b32 s31, v36, 9
; GFX9-NEXT: v_readlane_b32 s51, v36, 7
@@ -223620,11 +224508,16 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s13, s1
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-TRUE16-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB105_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s36, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s36, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -224164,8 +225057,6 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v34.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v33.h
; GFX11-TRUE16-NEXT: s_branch .LBB105_5
-; GFX11-TRUE16-NEXT: .LBB105_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB105_2
; GFX11-TRUE16-NEXT: .LBB105_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -224235,11 +225126,16 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s13, s1
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB105_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-FAKE16-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB105_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB105_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB105_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s36, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s36, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -224825,8 +225721,6 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_and_or_b32 v17, 0xffff0000, v32, v39
; GFX11-FAKE16-NEXT: v_and_or_b32 v16, 0xffff0000, v16, v48
; GFX11-FAKE16-NEXT: s_branch .LBB105_5
-; GFX11-FAKE16-NEXT: .LBB105_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB105_2
; GFX11-FAKE16-NEXT: .LBB105_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
@@ -225903,13 +226797,13 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: v_writelane_b32 v33, s31, 35
; SI-NEXT: s_lshr_b32 s5, s16, 16
; SI-NEXT: ; implicit-def: $vgpr34 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 s6, s17, 16
+; SI-NEXT: s_lshr_b32 s88, s17, 16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v34, s5, 0
-; SI-NEXT: s_lshr_b32 s8, s18, 16
-; SI-NEXT: v_writelane_b32 v34, s6, 1
+; SI-NEXT: s_lshr_b32 s89, s18, 16
+; SI-NEXT: v_writelane_b32 v34, s88, 1
; SI-NEXT: s_lshr_b32 s90, s19, 16
-; SI-NEXT: v_writelane_b32 v34, s8, 2
+; SI-NEXT: v_writelane_b32 v34, s89, 2
; SI-NEXT: s_lshr_b32 s91, s20, 16
; SI-NEXT: v_writelane_b32 v34, s90, 3
; SI-NEXT: s_lshr_b32 s92, s21, 16
@@ -225927,28 +226821,26 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_lshr_b32 s30, s27, 16
; SI-NEXT: v_writelane_b32 v34, vcc_hi, 10
; SI-NEXT: s_lshr_b32 s31, s28, 16
+; SI-NEXT: v_readfirstlane_b32 s79, v17
+; SI-NEXT: v_readfirstlane_b32 s77, v16
+; SI-NEXT: v_readfirstlane_b32 s75, v15
+; SI-NEXT: v_readfirstlane_b32 s73, v14
+; SI-NEXT: v_readfirstlane_b32 s63, v13
+; SI-NEXT: v_readfirstlane_b32 s61, v12
+; SI-NEXT: v_readfirstlane_b32 s59, v11
+; SI-NEXT: v_readfirstlane_b32 s57, v10
+; SI-NEXT: v_readfirstlane_b32 s47, v9
+; SI-NEXT: v_readfirstlane_b32 s45, v8
+; SI-NEXT: v_readfirstlane_b32 s43, v7
+; SI-NEXT: v_readfirstlane_b32 s41, v6
+; SI-NEXT: v_readfirstlane_b32 s15, v5
+; SI-NEXT: v_readfirstlane_b32 s13, v4
+; SI-NEXT: v_readfirstlane_b32 s11, v3
+; SI-NEXT: v_readfirstlane_b32 s9, v2
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: v_readfirstlane_b32 s99, v0
; SI-NEXT: v_writelane_b32 v34, s30, 11
; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: v_readfirstlane_b32 s89, v17
-; SI-NEXT: v_readfirstlane_b32 s79, v16
-; SI-NEXT: v_readfirstlane_b32 s77, v15
-; SI-NEXT: v_readfirstlane_b32 s75, v14
-; SI-NEXT: v_readfirstlane_b32 s73, v13
-; SI-NEXT: v_readfirstlane_b32 s63, v12
-; SI-NEXT: v_readfirstlane_b32 s61, v11
-; SI-NEXT: v_readfirstlane_b32 s59, v10
-; SI-NEXT: v_readfirstlane_b32 s57, v9
-; SI-NEXT: v_readfirstlane_b32 s47, v8
-; SI-NEXT: v_readfirstlane_b32 s45, v7
-; SI-NEXT: v_readfirstlane_b32 s43, v6
-; SI-NEXT: v_readfirstlane_b32 s41, v5
-; SI-NEXT: v_readfirstlane_b32 s15, v4
-; SI-NEXT: v_readfirstlane_b32 s13, v3
-; SI-NEXT: v_readfirstlane_b32 s11, v2
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: v_writelane_b32 v34, s31, 12
-; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s78, s79, 16
; SI-NEXT: s_lshr_b32 s76, s77, 16
; SI-NEXT: s_lshr_b32 s74, s75, 16
@@ -225964,128 +226856,126 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_lshr_b32 s14, s15, 16
; SI-NEXT: s_lshr_b32 s12, s13, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s7, 16
+; SI-NEXT: s_lshr_b32 s8, s9, 16
+; SI-NEXT: s_lshr_b32 s6, s7, 16
+; SI-NEXT: s_lshr_b32 s35, s99, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v34, s34, 13
+; SI-NEXT: v_writelane_b32 v34, s31, 12
; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: v_writelane_b32 v34, s34, 13
; SI-NEXT: v_writelane_b32 v34, s35, 14
-; SI-NEXT: v_writelane_b32 v34, s36, 15
; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s4, s16, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 17
+; SI-NEXT: v_writelane_b32 v34, s4, 18
; SI-NEXT: s_lshl_b32 s4, s5, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 16
+; SI-NEXT: v_writelane_b32 v34, s4, 17
; SI-NEXT: s_lshl_b32 s4, s17, 16
+; SI-NEXT: v_writelane_b32 v34, s4, 20
+; SI-NEXT: s_lshl_b32 s4, s88, 16
; SI-NEXT: v_writelane_b32 v34, s4, 19
-; SI-NEXT: s_lshl_b32 s4, s6, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 18
; SI-NEXT: s_lshl_b32 s4, s18, 16
+; SI-NEXT: v_writelane_b32 v34, s4, 22
+; SI-NEXT: s_lshl_b32 s4, s89, 16
; SI-NEXT: v_writelane_b32 v34, s4, 21
-; SI-NEXT: s_lshl_b32 s4, s8, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 20
; SI-NEXT: s_lshl_b32 s4, s19, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 23
+; SI-NEXT: v_writelane_b32 v34, s4, 24
; SI-NEXT: s_lshl_b32 s4, s90, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 22
+; SI-NEXT: v_writelane_b32 v34, s4, 23
; SI-NEXT: s_lshl_b32 s4, s20, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 25
+; SI-NEXT: v_writelane_b32 v34, s4, 26
; SI-NEXT: s_lshl_b32 s4, s91, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 24
+; SI-NEXT: v_writelane_b32 v34, s4, 25
; SI-NEXT: s_lshl_b32 s4, s21, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 27
+; SI-NEXT: v_writelane_b32 v34, s4, 28
; SI-NEXT: s_lshl_b32 s4, s92, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 26
+; SI-NEXT: v_writelane_b32 v34, s4, 27
; SI-NEXT: s_lshl_b32 s4, s22, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 29
+; SI-NEXT: v_writelane_b32 v34, s4, 30
; SI-NEXT: s_lshl_b32 s4, s93, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 28
+; SI-NEXT: v_writelane_b32 v34, s4, 29
; SI-NEXT: s_lshl_b32 s4, s23, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 31
+; SI-NEXT: v_writelane_b32 v34, s4, 32
; SI-NEXT: s_lshl_b32 s4, s94, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 30
+; SI-NEXT: v_writelane_b32 v34, s4, 31
; SI-NEXT: s_lshl_b32 s4, s24, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 33
+; SI-NEXT: v_writelane_b32 v34, s4, 16
; SI-NEXT: s_lshl_b32 s4, s95, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 32
+; SI-NEXT: v_writelane_b32 v34, s4, 33
; SI-NEXT: s_lshl_b32 s4, s25, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 35
-; SI-NEXT: s_lshl_b32 s4, vcc_lo, 16
-; SI-NEXT: v_writelane_b32 v34, s4, 34
-; SI-NEXT: s_lshl_b32 s99, s26, 16
-; SI-NEXT: s_lshl_b32 s8, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s87, s27, 16
-; SI-NEXT: s_lshl_b32 s6, s30, 16
-; SI-NEXT: s_lshl_b32 s85, s28, 16
-; SI-NEXT: s_lshl_b32 s98, s31, 16
-; SI-NEXT: s_lshl_b32 s83, s29, 16
-; SI-NEXT: s_lshl_b32 s97, s34, 16
-; SI-NEXT: s_lshl_b32 s81, s7, 16
-; SI-NEXT: s_lshl_b32 s96, s35, 16
-; SI-NEXT: s_lshl_b32 s71, s9, 16
-; SI-NEXT: s_lshl_b32 s86, s36, 16
-; SI-NEXT: s_lshl_b32 s80, s11, 16
-; SI-NEXT: s_lshl_b32 s84, s10, 16
-; SI-NEXT: s_lshl_b32 s67, s13, 16
-; SI-NEXT: s_lshl_b32 s82, s12, 16
-; SI-NEXT: s_lshl_b32 s65, s15, 16
-; SI-NEXT: s_lshl_b32 s69, s14, 16
-; SI-NEXT: s_lshl_b32 s55, s41, 16
-; SI-NEXT: s_lshl_b32 s70, s40, 16
-; SI-NEXT: s_lshl_b32 s53, s43, 16
-; SI-NEXT: s_lshl_b32 s68, s42, 16
-; SI-NEXT: s_lshl_b32 s51, s45, 16
-; SI-NEXT: s_lshl_b32 s66, s44, 16
-; SI-NEXT: s_lshl_b32 s49, s47, 16
-; SI-NEXT: s_lshl_b32 s64, s46, 16
-; SI-NEXT: s_lshl_b32 s39, s57, 16
-; SI-NEXT: s_lshl_b32 s54, s56, 16
-; SI-NEXT: s_lshl_b32 s37, s59, 16
-; SI-NEXT: s_lshl_b32 s52, s58, 16
-; SI-NEXT: s_lshl_b32 s35, s61, 16
-; SI-NEXT: s_lshl_b32 s50, s60, 16
-; SI-NEXT: s_lshl_b32 s31, s63, 16
-; SI-NEXT: s_lshl_b32 s48, s62, 16
-; SI-NEXT: s_lshl_b32 s95, s73, 16
-; SI-NEXT: s_lshl_b32 s38, s72, 16
-; SI-NEXT: s_lshl_b32 s34, s75, 16
-; SI-NEXT: s_lshl_b32 s36, s74, 16
-; SI-NEXT: s_lshl_b32 s92, s77, 16
-; SI-NEXT: s_lshl_b32 s30, s76, 16
-; SI-NEXT: s_lshl_b32 s93, s79, 16
-; SI-NEXT: s_lshl_b32 s94, s78, 16
-; SI-NEXT: s_lshl_b32 s90, s89, 16
-; SI-NEXT: s_lshl_b32 s91, s88, 16
+; SI-NEXT: v_writelane_b32 v34, s4, 15
+; SI-NEXT: s_lshl_b32 s98, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s96, s26, 16
+; SI-NEXT: s_lshl_b32 s97, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s85, s27, 16
+; SI-NEXT: s_lshl_b32 s87, s30, 16
+; SI-NEXT: s_lshl_b32 s81, s28, 16
+; SI-NEXT: s_lshl_b32 s86, s31, 16
+; SI-NEXT: s_lshl_b32 s71, s29, 16
+; SI-NEXT: s_lshl_b32 s84, s34, 16
+; SI-NEXT: s_lshl_b32 s69, s99, 16
+; SI-NEXT: s_lshl_b32 s83, s35, 16
+; SI-NEXT: s_lshl_b32 s67, s7, 16
+; SI-NEXT: s_lshl_b32 s82, s6, 16
+; SI-NEXT: s_lshl_b32 s68, s9, 16
+; SI-NEXT: s_lshl_b32 s80, s8, 16
+; SI-NEXT: s_lshl_b32 s55, s11, 16
+; SI-NEXT: s_lshl_b32 s70, s10, 16
+; SI-NEXT: s_lshl_b32 s53, s13, 16
+; SI-NEXT: s_lshl_b32 s65, s12, 16
+; SI-NEXT: s_lshl_b32 s51, s15, 16
+; SI-NEXT: s_lshl_b32 s66, s14, 16
+; SI-NEXT: s_lshl_b32 s49, s41, 16
+; SI-NEXT: s_lshl_b32 s64, s40, 16
+; SI-NEXT: s_lshl_b32 s39, s43, 16
+; SI-NEXT: s_lshl_b32 s54, s42, 16
+; SI-NEXT: s_lshl_b32 s37, s45, 16
+; SI-NEXT: s_lshl_b32 s52, s44, 16
+; SI-NEXT: s_lshl_b32 s35, s47, 16
+; SI-NEXT: s_lshl_b32 s50, s46, 16
+; SI-NEXT: s_lshl_b32 s31, s57, 16
+; SI-NEXT: s_lshl_b32 s48, s56, 16
+; SI-NEXT: s_lshl_b32 vcc_hi, s59, 16
+; SI-NEXT: s_lshl_b32 s38, s58, 16
+; SI-NEXT: s_lshl_b32 s95, s61, 16
+; SI-NEXT: s_lshl_b32 s36, s60, 16
+; SI-NEXT: s_lshl_b32 s93, s63, 16
+; SI-NEXT: s_lshl_b32 s34, s62, 16
+; SI-NEXT: s_lshl_b32 vcc_lo, s73, 16
+; SI-NEXT: s_lshl_b32 s30, s72, 16
+; SI-NEXT: s_lshl_b32 s90, s75, 16
+; SI-NEXT: s_lshl_b32 s94, s74, 16
+; SI-NEXT: s_lshl_b32 s91, s77, 16
+; SI-NEXT: s_lshl_b32 s92, s76, 16
+; SI-NEXT: s_lshl_b32 s88, s79, 16
+; SI-NEXT: s_lshl_b32 s89, s78, 16
; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_branch .LBB107_3
; SI-NEXT: .LBB107_2:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr96
; SI-NEXT: ; implicit-def: $sgpr97
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr87
; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr71
; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr71
; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr83
; SI-NEXT: ; implicit-def: $sgpr67
; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr55
; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr65
; SI-NEXT: ; implicit-def: $sgpr51
; SI-NEXT: ; implicit-def: $sgpr66
; SI-NEXT: ; implicit-def: $sgpr49
@@ -226098,61 +226988,68 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr31
; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $vcc_hi
; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr95
; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr30
; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; kill: killed $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; kill: killed $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr88
; SI-NEXT: .LBB107_3: ; %Flow
-; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; SI-NEXT: s_mov_b32 s4, s90
-; SI-NEXT: s_mov_b32 s5, s93
-; SI-NEXT: s_mov_b32 s90, s92
-; SI-NEXT: s_mov_b32 s92, s34
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_mov_b32 s4, s88
+; SI-NEXT: s_mov_b32 s5, s91
+; SI-NEXT: s_mov_b32 s88, s90
+; SI-NEXT: s_mov_b32 s90, vcc_lo
+; SI-NEXT: s_mov_b32 s91, s93
; SI-NEXT: s_mov_b32 s93, s95
-; SI-NEXT: s_mov_b32 s95, s31
+; SI-NEXT: s_mov_b32 s95, vcc_hi
+; SI-NEXT: s_mov_b32 vcc_lo, s30
+; SI-NEXT: s_mov_b32 vcc_hi, s31
+; SI-NEXT: s_mov_b32 s30, s34
; SI-NEXT: s_mov_b32 s31, s35
; SI-NEXT: s_mov_b32 s34, s36
; SI-NEXT: s_mov_b32 s35, s37
@@ -226167,150 +227064,141 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_mov_b32 s52, s54
; SI-NEXT: s_mov_b32 s53, s55
; SI-NEXT: s_mov_b32 s54, s64
-; SI-NEXT: s_mov_b32 s55, s65
+; SI-NEXT: s_mov_b32 s55, s68
; SI-NEXT: s_mov_b32 s64, s66
-; SI-NEXT: s_mov_b32 s65, s67
-; SI-NEXT: s_mov_b32 s66, s68
-; SI-NEXT: s_mov_b32 s67, s80
-; SI-NEXT: s_mov_b32 s68, s70
-; SI-NEXT: s_mov_b32 s70, s82
-; SI-NEXT: s_mov_b32 s80, s6
-; SI-NEXT: s_mov_b32 s82, s8
-; SI-NEXT: s_cbranch_vccnz .LBB107_5
+; SI-NEXT: s_mov_b32 s66, s70
+; SI-NEXT: v_readlane_b32 s68, v34, 15
+; SI-NEXT: v_readlane_b32 s70, v34, 16
+; SI-NEXT: s_cbranch_scc1 .LBB107_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: v_readlane_b32 s6, v34, 15
; SI-NEXT: s_add_i32 s11, s11, 3
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_add_i32 s7, s7, 3
-; SI-NEXT: v_readlane_b32 s6, v34, 14
; SI-NEXT: s_add_i32 s13, s13, 3
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: v_readlane_b32 s9, v34, 13
+; SI-NEXT: s_add_i32 s99, s99, 3
+; SI-NEXT: v_readlane_b32 s9, v34, 14
; SI-NEXT: s_add_i32 s15, s15, 3
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_or_b32 s10, s10, s11
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_and_b32 s7, s29, 0xffff
+; SI-NEXT: s_and_b32 s7, s99, 0xffff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: v_readlane_b32 s11, v34, 12
+; SI-NEXT: s_add_i32 s29, s29, 3
+; SI-NEXT: v_readlane_b32 s11, v34, 13
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: s_or_b32 s7, s9, s7
-; SI-NEXT: s_and_b32 s9, s28, 0xffff
+; SI-NEXT: s_and_b32 s9, s29, 0xffff
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: v_readlane_b32 s13, v34, 11
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: v_readlane_b32 s13, v34, 12
; SI-NEXT: s_or_b32 s14, s14, s15
; SI-NEXT: s_or_b32 s9, s11, s9
-; SI-NEXT: s_and_b32 s11, s27, 0xffff
+; SI-NEXT: s_and_b32 s11, s28, 0xffff
; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: v_readlane_b32 s15, v34, 10
+; SI-NEXT: s_add_i32 s27, s27, 3
+; SI-NEXT: v_readlane_b32 s15, v34, 11
; SI-NEXT: s_or_b32 s11, s13, s11
-; SI-NEXT: s_and_b32 s13, s26, 0xffff
+; SI-NEXT: s_and_b32 s13, s27, 0xffff
; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_add_i32 s25, s25, 3
+; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s13, s15, s13
-; SI-NEXT: s_and_b32 s15, s25, 0xffff
-; SI-NEXT: v_readlane_b32 s25, v34, 9
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s15, s25, s15
+; SI-NEXT: s_and_b32 s15, s26, 0xffff
+; SI-NEXT: v_readlane_b32 s26, v34, 10
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s15, s26, s15
+; SI-NEXT: s_add_i32 s25, s25, 3
+; SI-NEXT: v_readlane_b32 s26, v34, 9
+; SI-NEXT: s_and_b32 s25, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s25, s26, s25
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 8
+; SI-NEXT: v_readlane_b32 s26, v34, 8
; SI-NEXT: s_and_b32 s24, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s24, s25, s24
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s24, s26, s24
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 7
+; SI-NEXT: v_readlane_b32 s26, v34, 7
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s23, s25, s23
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s23, s26, s23
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 6
+; SI-NEXT: v_readlane_b32 s26, v34, 6
; SI-NEXT: s_and_b32 s22, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s22, s25, s22
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s22, s26, s22
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 5
+; SI-NEXT: v_readlane_b32 s26, v34, 5
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s21, s25, s21
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s21, s26, s21
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 4
+; SI-NEXT: v_readlane_b32 s26, v34, 4
; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s20, s25, s20
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s20, s26, s20
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 3
+; SI-NEXT: v_readlane_b32 s26, v34, 3
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s19, s25, s19
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s19, s26, s19
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 2
+; SI-NEXT: v_readlane_b32 s26, v34, 2
; SI-NEXT: s_and_b32 s18, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s18, s25, s18
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s18, s26, s18
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 1
+; SI-NEXT: v_readlane_b32 s26, v34, 1
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s17, s25, s17
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s17, s26, s17
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s25, v34, 0
+; SI-NEXT: v_readlane_b32 s26, v34, 0
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s25, 16
-; SI-NEXT: s_or_b32 s16, s25, s16
+; SI-NEXT: s_lshl_b32 s26, s26, 16
+; SI-NEXT: s_or_b32 s16, s26, s16
; SI-NEXT: s_add_i32 s16, s16, 0x30000
-; SI-NEXT: s_and_b32 s25, s16, 0xffff0000
+; SI-NEXT: s_and_b32 s26, s16, 0xffff0000
; SI-NEXT: s_add_i32 s17, s17, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s25, 16
+; SI-NEXT: v_writelane_b32 v34, s26, 17
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: v_writelane_b32 v34, s16, 17
+; SI-NEXT: v_writelane_b32 v34, s16, 18
; SI-NEXT: s_and_b32 s16, s17, 0xffff0000
; SI-NEXT: s_add_i32 s18, s18, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 18
-; SI-NEXT: s_lshl_b32 s16, s17, 16
; SI-NEXT: v_writelane_b32 v34, s16, 19
+; SI-NEXT: s_lshl_b32 s16, s17, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 20
; SI-NEXT: s_and_b32 s16, s18, 0xffff0000
; SI-NEXT: s_add_i32 s19, s19, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 20
-; SI-NEXT: s_lshl_b32 s16, s18, 16
; SI-NEXT: v_writelane_b32 v34, s16, 21
+; SI-NEXT: s_lshl_b32 s16, s18, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 22
; SI-NEXT: s_and_b32 s16, s19, 0xffff0000
; SI-NEXT: s_add_i32 s20, s20, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 22
-; SI-NEXT: s_lshl_b32 s16, s19, 16
; SI-NEXT: v_writelane_b32 v34, s16, 23
+; SI-NEXT: s_lshl_b32 s16, s19, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 24
; SI-NEXT: s_and_b32 s16, s20, 0xffff0000
; SI-NEXT: s_add_i32 s21, s21, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 24
-; SI-NEXT: s_lshl_b32 s16, s20, 16
; SI-NEXT: v_writelane_b32 v34, s16, 25
+; SI-NEXT: s_lshl_b32 s16, s20, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 26
; SI-NEXT: s_and_b32 s16, s21, 0xffff0000
+; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_add_i32 s22, s22, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 26
-; SI-NEXT: s_lshl_b32 s16, s21, 16
; SI-NEXT: v_writelane_b32 v34, s16, 27
-; SI-NEXT: s_and_b32 s16, s22, 0xffff0000
-; SI-NEXT: s_add_i32 s23, s23, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 28
-; SI-NEXT: s_lshl_b32 s16, s22, 16
-; SI-NEXT: s_add_i32 s89, s89, 3
-; SI-NEXT: v_writelane_b32 v34, s16, 29
-; SI-NEXT: s_and_b32 s16, s23, 0xffff0000
-; SI-NEXT: s_and_b32 s4, s89, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s88, 16
-; SI-NEXT: s_add_i32 s79, s79, 3
+; SI-NEXT: s_lshl_b32 s16, s21, 16
+; SI-NEXT: s_and_b32 s4, s79, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_add_i32 s75, s75, 3
; SI-NEXT: s_add_i32 s73, s73, 3
@@ -226322,13 +227210,10 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_add_i32 s45, s45, 3
; SI-NEXT: s_add_i32 s43, s43, 3
; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_add_i32 s24, s24, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 30
-; SI-NEXT: s_lshl_b32 s16, s23, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 28
+; SI-NEXT: s_and_b32 s16, s22, 0xffff0000
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s79, 0xffff
-; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_and_b32 s77, s77, 0xffff
+; SI-NEXT: s_and_b32 s5, s77, 0xffff
; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_and_b32 s75, s75, 0xffff
; SI-NEXT: s_lshl_b32 s74, s74, 16
@@ -226350,10 +227235,10 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_and_b32 s41, s41, 0xffff
; SI-NEXT: s_lshl_b32 s40, s40, 16
-; SI-NEXT: v_writelane_b32 v34, s16, 31
-; SI-NEXT: s_and_b32 s16, s24, 0xffff0000
-; SI-NEXT: s_or_b32 s5, s78, s5
-; SI-NEXT: s_or_b32 s76, s76, s77
+; SI-NEXT: s_add_i32 s23, s23, 0x30000
+; SI-NEXT: v_writelane_b32 v34, s16, 29
+; SI-NEXT: s_lshl_b32 s16, s22, 16
+; SI-NEXT: s_or_b32 s5, s76, s5
; SI-NEXT: s_or_b32 s74, s74, s75
; SI-NEXT: s_or_b32 s72, s72, s73
; SI-NEXT: s_or_b32 s62, s62, s63
@@ -226364,12 +227249,10 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_or_b32 s44, s44, s45
; SI-NEXT: s_or_b32 s42, s42, s43
; SI-NEXT: s_or_b32 s40, s40, s41
-; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 32
-; SI-NEXT: s_lshl_b32 s16, s24, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 30
+; SI-NEXT: s_and_b32 s16, s23, 0xffff0000
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_add_i32 s76, s76, 0x30000
; SI-NEXT: s_add_i32 s74, s74, 0x30000
; SI-NEXT: s_add_i32 s72, s72, 0x30000
; SI-NEXT: s_add_i32 s62, s62, 0x30000
@@ -226389,201 +227272,204 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: v_writelane_b32 v34, s16, 33
-; SI-NEXT: s_and_b32 s16, s15, 0xffff0000
-; SI-NEXT: v_writelane_b32 v34, s16, 34
-; SI-NEXT: s_lshl_b32 s15, s15, 16
-; SI-NEXT: s_and_b32 s82, s13, 0xffff0000
-; SI-NEXT: s_lshl_b32 s99, s13, 16
-; SI-NEXT: s_and_b32 s80, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s87, s11, 16
-; SI-NEXT: s_and_b32 s98, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s85, s9, 16
-; SI-NEXT: s_and_b32 s97, s7, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, s7, 16
-; SI-NEXT: s_and_b32 s96, s6, 0xffff0000
-; SI-NEXT: s_lshl_b32 s81, s6, 16
-; SI-NEXT: s_and_b32 s86, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s8, 16
-; SI-NEXT: s_and_b32 s84, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s67, s10, 16
-; SI-NEXT: s_and_b32 s70, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s65, s12, 16
-; SI-NEXT: s_and_b32 s69, s14, 0xffff0000
-; SI-NEXT: s_lshl_b32 s55, s14, 16
-; SI-NEXT: s_and_b32 s68, s40, 0xffff0000
-; SI-NEXT: s_lshl_b32 s53, s40, 16
-; SI-NEXT: s_and_b32 s66, s42, 0xffff0000
-; SI-NEXT: s_lshl_b32 s51, s42, 16
-; SI-NEXT: s_and_b32 s64, s44, 0xffff0000
-; SI-NEXT: s_lshl_b32 s49, s44, 16
-; SI-NEXT: s_and_b32 s54, s46, 0xffff0000
-; SI-NEXT: s_lshl_b32 s39, s46, 16
-; SI-NEXT: s_and_b32 s52, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s37, s56, 16
-; SI-NEXT: s_and_b32 s50, s58, 0xffff0000
-; SI-NEXT: s_lshl_b32 s35, s58, 16
-; SI-NEXT: s_and_b32 s48, s60, 0xffff0000
-; SI-NEXT: s_lshl_b32 s31, s60, 16
-; SI-NEXT: s_and_b32 s38, s62, 0xffff0000
-; SI-NEXT: s_lshl_b32 s95, s62, 16
-; SI-NEXT: s_and_b32 s36, s72, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s72, 16
-; SI-NEXT: s_and_b32 s34, s74, 0xffff0000
-; SI-NEXT: s_lshl_b32 s92, s74, 16
-; SI-NEXT: s_and_b32 s30, s76, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s76, 16
-; SI-NEXT: s_and_b32 s94, s5, 0xffff0000
+; SI-NEXT: s_add_i32 s15, s15, 0x30000
+; SI-NEXT: s_add_i32 s25, s25, 0x30000
+; SI-NEXT: s_add_i32 s24, s24, 0x30000
+; SI-NEXT: v_writelane_b32 v34, s16, 31
+; SI-NEXT: s_lshl_b32 s16, s23, 16
+; SI-NEXT: v_writelane_b32 v34, s16, 32
+; SI-NEXT: s_and_b32 s16, s24, 0xffff0000
+; SI-NEXT: s_lshl_b32 s70, s24, 16
+; SI-NEXT: s_and_b32 s98, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s68, s25, 16
+; SI-NEXT: s_and_b32 s97, s15, 0xffff0000
+; SI-NEXT: s_lshl_b32 s96, s15, 16
+; SI-NEXT: s_and_b32 s87, s13, 0xffff0000
+; SI-NEXT: s_lshl_b32 s85, s13, 16
+; SI-NEXT: s_and_b32 s86, s11, 0xffff0000
+; SI-NEXT: s_lshl_b32 s81, s11, 16
+; SI-NEXT: s_and_b32 s84, s9, 0xffff0000
+; SI-NEXT: s_lshl_b32 s71, s9, 16
+; SI-NEXT: s_and_b32 s83, s7, 0xffff0000
+; SI-NEXT: s_lshl_b32 s69, s7, 16
+; SI-NEXT: s_and_b32 s82, s6, 0xffff0000
+; SI-NEXT: s_lshl_b32 s67, s6, 16
+; SI-NEXT: s_and_b32 s80, s8, 0xffff0000
+; SI-NEXT: s_lshl_b32 s55, s8, 16
+; SI-NEXT: s_and_b32 s66, s10, 0xffff0000
+; SI-NEXT: s_lshl_b32 s53, s10, 16
+; SI-NEXT: s_and_b32 s65, s12, 0xffff0000
+; SI-NEXT: s_lshl_b32 s51, s12, 16
+; SI-NEXT: s_and_b32 s64, s14, 0xffff0000
+; SI-NEXT: s_lshl_b32 s49, s14, 16
+; SI-NEXT: s_and_b32 s54, s40, 0xffff0000
+; SI-NEXT: s_lshl_b32 s39, s40, 16
+; SI-NEXT: s_and_b32 s52, s42, 0xffff0000
+; SI-NEXT: s_lshl_b32 s37, s42, 16
+; SI-NEXT: s_and_b32 s50, s44, 0xffff0000
+; SI-NEXT: s_lshl_b32 s35, s44, 16
+; SI-NEXT: s_and_b32 s48, s46, 0xffff0000
+; SI-NEXT: s_lshl_b32 s31, s46, 16
+; SI-NEXT: s_and_b32 s38, s56, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_hi, s56, 16
+; SI-NEXT: s_and_b32 s36, s58, 0xffff0000
+; SI-NEXT: s_lshl_b32 s95, s58, 16
+; SI-NEXT: s_and_b32 s34, s60, 0xffff0000
+; SI-NEXT: s_lshl_b32 s93, s60, 16
+; SI-NEXT: s_and_b32 s30, s62, 0xffff0000
+; SI-NEXT: s_lshl_b32 s91, s62, 16
+; SI-NEXT: s_and_b32 vcc_lo, s72, 0xffff0000
+; SI-NEXT: s_lshl_b32 s90, s72, 16
+; SI-NEXT: s_and_b32 s94, s74, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s74, 16
+; SI-NEXT: s_and_b32 s92, s5, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_and_b32 s91, s4, 0xffff0000
+; SI-NEXT: s_and_b32 s89, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_writelane_b32 v34, s15, 35
+; SI-NEXT: v_writelane_b32 v34, s16, 33
; SI-NEXT: .LBB107_5: ; %end
-; SI-NEXT: v_readlane_b32 s6, v34, 16
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 17
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 18
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 18
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 19
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 20
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 20
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 21
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 22
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 22
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 23
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 24
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 24
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 25
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 26
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 26
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 27
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 28
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 28
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 29
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 30
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 30
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 31
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
+; SI-NEXT: v_readlane_b32 s6, v34, 32
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 32
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
; SI-NEXT: v_readlane_b32 s6, v34, 33
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s70
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_readlane_b32 s6, v34, 34
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s6
-; SI-NEXT: v_readlane_b32 s6, v34, 35
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s98
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s68
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s82
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s97
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s99
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s96
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s80
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s87
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s85
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s98
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s86
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s85
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s81
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s97
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s84
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s83
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s71
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s96
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s83
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s81
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s69
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s86
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s82
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s71
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s67
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s84
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s80
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s67
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s55
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s70
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s66
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s65
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s53
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s69
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s65
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s55
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s51
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s68
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s64
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s53
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s49
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s66
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s54
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s51
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s39
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s64
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s52
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s49
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s37
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s54
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s50
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s39
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s35
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s52
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s48
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s31
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s50
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s38
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, vcc_hi
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s48
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s36
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s31
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s95
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s38
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s34
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s93
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s36
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s30
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s93
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s91
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s34
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, vcc_lo
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v28
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s92
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s90
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s94
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s88
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s92
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
; SI-NEXT: v_mul_f32_e64 v30, 1.0, s5
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s89
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v31
; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
; SI-NEXT: v_readlane_b32 s30, v33, 34
@@ -226664,10 +227550,18 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB107_4
+; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB107_3
-; VI-NEXT: .LBB107_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB107_3
+; VI-NEXT: .LBB107_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB107_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB107_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s47, 0xffff0000
; VI-NEXT: s_add_i32 s5, s47, 3
; VI-NEXT: s_and_b32 s47, s46, 0xffff0000
@@ -226828,7 +227722,7 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; VI-NEXT: s_add_i32 s45, s45, 0x30000
; VI-NEXT: s_add_i32 s46, s46, 0x30000
; VI-NEXT: s_add_i32 s47, s4, 0x30000
-; VI-NEXT: .LBB107_3: ; %end
+; VI-NEXT: .LBB107_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 6
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -226874,8 +227768,6 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB107_4:
-; VI-NEXT: s_branch .LBB107_2
;
; GFX9-LABEL: bitcast_v64i16_to_v64bf16_scalar:
; GFX9: ; %bb.0:
@@ -226913,10 +227805,18 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB107_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB107_4
-; GFX9-NEXT: .LBB107_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB107_3
+; GFX9-NEXT: .LBB107_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB107_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -226949,10 +227849,8 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; GFX9-NEXT: v_pk_add_u16 v18, s38, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s36, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB107_5
-; GFX9-NEXT: .LBB107_3:
-; GFX9-NEXT: s_branch .LBB107_2
-; GFX9-NEXT: .LBB107_4:
+; GFX9-NEXT: s_branch .LBB107_6
+; GFX9-NEXT: .LBB107_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -226985,7 +227883,7 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB107_5: ; %end
+; GFX9-NEXT: .LBB107_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -227039,11 +227937,16 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB107_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB107_4
-; GFX11-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB107_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -227077,8 +227980,6 @@ define inreg <64 x bfloat> @bitcast_v64i16_to_v64bf16_scalar(<64 x i16> inreg %a
; GFX11-NEXT: v_pk_add_u16 v17, s37, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s36, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB107_5
-; GFX11-NEXT: .LBB107_3:
-; GFX11-NEXT: s_branch .LBB107_2
; GFX11-NEXT: .LBB107_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -227868,10 +228769,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v63, s35, 1
; SI-NEXT: v_writelane_b32 v63, s36, 2
; SI-NEXT: v_writelane_b32 v63, s37, 3
-; SI-NEXT: v_writelane_b32 v63, s38, 4
-; SI-NEXT: v_writelane_b32 v63, s39, 5
-; SI-NEXT: v_writelane_b32 v63, s30, 6
-; SI-NEXT: v_writelane_b32 v63, s31, 7
+; SI-NEXT: v_writelane_b32 v63, s30, 4
+; SI-NEXT: v_writelane_b32 v63, s31, 5
; SI-NEXT: v_readfirstlane_b32 s10, v17
; SI-NEXT: v_readfirstlane_b32 s6, v16
; SI-NEXT: v_readfirstlane_b32 s11, v15
@@ -227887,23 +228786,23 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s46, v5
; SI-NEXT: v_readfirstlane_b32 s92, v4
; SI-NEXT: v_readfirstlane_b32 s45, v3
-; SI-NEXT: v_readfirstlane_b32 s30, v2
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: v_readfirstlane_b32 s44, v1
-; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: v_readfirstlane_b32 s31, v0
; SI-NEXT: s_lshr_b32 s43, s29, 16
; SI-NEXT: s_lshr_b32 s93, s28, 16
; SI-NEXT: s_lshr_b32 s42, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
; SI-NEXT: s_lshr_b32 s41, s25, 16
-; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s30, s24, 16
; SI-NEXT: s_lshr_b32 s40, s23, 16
-; SI-NEXT: s_lshr_b32 s36, s22, 16
+; SI-NEXT: s_lshr_b32 s34, s22, 16
; SI-NEXT: s_lshr_b32 s15, s21, 16
-; SI-NEXT: s_lshr_b32 s37, s20, 16
+; SI-NEXT: s_lshr_b32 s35, s20, 16
; SI-NEXT: s_lshr_b32 s14, s19, 16
-; SI-NEXT: s_lshr_b32 s38, s18, 16
+; SI-NEXT: s_lshr_b32 s36, s18, 16
; SI-NEXT: s_lshr_b32 s13, s17, 16
-; SI-NEXT: s_lshr_b32 s39, s16, 16
+; SI-NEXT: s_lshr_b32 s37, s16, 16
; SI-NEXT: s_lshr_b32 s88, s10, 16
; SI-NEXT: s_lshr_b32 s61, s6, 16
; SI-NEXT: s_lshr_b32 s76, s11, 16
@@ -227919,18 +228818,26 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s62, s46, 16
; SI-NEXT: s_lshr_b32 s91, s92, 16
; SI-NEXT: s_lshr_b32 s60, s45, 16
-; SI-NEXT: s_lshr_b32 s94, s30, 16
+; SI-NEXT: s_lshr_b32 s94, vcc_lo, 16
; SI-NEXT: s_lshr_b32 s58, s44, 16
-; SI-NEXT: s_lshr_b32 s95, s35, 16
+; SI-NEXT: s_lshr_b32 s95, s31, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB109_3
+; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB109_4
-; SI-NEXT: .LBB109_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s39
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB109_3
+; SI-NEXT: .LBB109_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB109_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB109_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s37
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s38
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s36
; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -227938,7 +228845,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s37
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
; SI-NEXT: v_or_b32_e32 v40, v1, v38
; SI-NEXT: v_cvt_f16_f32_e32 v1, v2
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
@@ -227950,7 +228857,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v1, s20
; SI-NEXT: v_or_b32_e32 v2, v2, v60
; SI-NEXT: v_lshlrev_b32_e32 v56, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s36
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s34
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v2, s22
@@ -227961,14 +228868,14 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_or_b32_e32 v8, v1, v56
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s30
; SI-NEXT: v_lshlrev_b32_e32 v45, 16, v0
; SI-NEXT: v_or_b32_e32 v2, v2, v45
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v2, vcc_hi
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -227993,7 +228900,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v41, 16, v0
; SI-NEXT: v_or_b32_e32 v17, v2, v41
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
@@ -228001,7 +228908,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_lshlrev_b32_e32 v52, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v1, vcc_lo
; SI-NEXT: v_lshlrev_b32_e32 v49, 16, v2
; SI-NEXT: v_cvt_f32_f16_e32 v2, s91
; SI-NEXT: v_or_b32_e32 v21, v0, v52
@@ -228263,10 +229170,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v50, v7
-; SI-NEXT: s_branch .LBB109_5
-; SI-NEXT: .LBB109_3:
-; SI-NEXT: s_branch .LBB109_2
-; SI-NEXT: .LBB109_4:
+; SI-NEXT: s_branch .LBB109_6
+; SI-NEXT: .LBB109_5:
; SI-NEXT: v_mov_b32_e32 v6, s20
; SI-NEXT: v_mov_b32_e32 v7, s25
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -228302,8 +229207,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v24, s12
; SI-NEXT: v_mov_b32_e32 v48, s90
; SI-NEXT: v_mov_b32_e32 v20, s92
-; SI-NEXT: v_mov_b32_e32 v26, s30
-; SI-NEXT: v_mov_b32_e32 v17, s35
+; SI-NEXT: v_mov_b32_e32 v26, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v17, s31
; SI-NEXT: v_mov_b32_e32 v14, s28
; SI-NEXT: v_mov_b32_e32 v12, s26
; SI-NEXT: v_mov_b32_e32 v10, s24
@@ -228325,12 +229230,12 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s11
; SI-NEXT: v_mov_b32_e32 v31, s10
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s39
-; SI-NEXT: v_mov_b32_e32 v60, s38
-; SI-NEXT: v_mov_b32_e32 v61, s37
-; SI-NEXT: v_mov_b32_e32 v56, s36
-; SI-NEXT: v_mov_b32_e32 v13, s34
-; SI-NEXT: v_mov_b32_e32 v44, s31
+; SI-NEXT: v_mov_b32_e32 v0, s37
+; SI-NEXT: v_mov_b32_e32 v60, s36
+; SI-NEXT: v_mov_b32_e32 v61, s35
+; SI-NEXT: v_mov_b32_e32 v56, s34
+; SI-NEXT: v_mov_b32_e32 v13, s30
+; SI-NEXT: v_mov_b32_e32 v44, vcc_hi
; SI-NEXT: v_mov_b32_e32 v42, s93
; SI-NEXT: v_mov_b32_e32 v36, s89
; SI-NEXT: v_mov_b32_e32 v58, s74
@@ -228350,7 +229255,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v7, s77
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: .LBB109_5: ; %end
+; SI-NEXT: .LBB109_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
@@ -228394,10 +229299,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
-; SI-NEXT: v_readlane_b32 s30, v63, 6
-; SI-NEXT: v_readlane_b32 s31, v63, 7
-; SI-NEXT: v_readlane_b32 s39, v63, 5
-; SI-NEXT: v_readlane_b32 s38, v63, 4
+; SI-NEXT: v_readlane_b32 s30, v63, 4
+; SI-NEXT: v_readlane_b32 s31, v63, 5
; SI-NEXT: v_readlane_b32 s37, v63, 3
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
@@ -228535,10 +229438,18 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB109_3
+; VI-NEXT: s_cbranch_scc0 .LBB109_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB109_4
-; VI-NEXT: .LBB109_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB109_3
+; VI-NEXT: .LBB109_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB109_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB109_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s50, 16
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_lshr_b32 s4, s30, 16
@@ -228700,10 +229611,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v32, v32, v16 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s36, v16
; VI-NEXT: v_or_b32_e32 v16, v16, v32
-; VI-NEXT: s_branch .LBB109_5
-; VI-NEXT: .LBB109_3:
-; VI-NEXT: s_branch .LBB109_2
-; VI-NEXT: .LBB109_4:
+; VI-NEXT: s_branch .LBB109_6
+; VI-NEXT: .LBB109_5:
; VI-NEXT: v_mov_b32_e32 v16, s36
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v17, s37
@@ -228736,7 +229645,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB109_5: ; %end
+; VI-NEXT: .LBB109_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v33, 8
; VI-NEXT: v_readlane_b32 s31, v33, 9
; VI-NEXT: v_readlane_b32 s51, v33, 7
@@ -228789,10 +229698,18 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB109_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB109_4
-; GFX9-NEXT: .LBB109_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB109_3
+; GFX9-NEXT: .LBB109_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB109_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v16, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v16 op_sel_hi:[1,0]
@@ -228826,10 +229743,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v18, s38, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s37, v16 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s36, v16 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB109_5
-; GFX9-NEXT: .LBB109_3:
-; GFX9-NEXT: s_branch .LBB109_2
-; GFX9-NEXT: .LBB109_4:
+; GFX9-NEXT: s_branch .LBB109_6
+; GFX9-NEXT: .LBB109_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -228862,7 +229777,7 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB109_5: ; %end
+; GFX9-NEXT: .LBB109_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -228916,11 +229831,16 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB109_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB109_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB109_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB109_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -228954,8 +229874,6 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s37 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s36 op_sel_hi:[0,1]
; GFX11-NEXT: s_branch .LBB109_5
-; GFX11-NEXT: .LBB109_3:
-; GFX11-NEXT: s_branch .LBB109_2
; GFX11-NEXT: .LBB109_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
@@ -230234,87 +231152,86 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s99, 33
; SI-NEXT: v_writelane_b32 v32, s30, 34
; SI-NEXT: v_writelane_b32 v32, s31, 35
-; SI-NEXT: s_lshr_b32 s4, s21, 16
; SI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; SI-NEXT: s_lshr_b32 s30, s20, 16
+; SI-NEXT: s_lshr_b32 s4, s17, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v33, s4, 0
-; SI-NEXT: v_writelane_b32 v33, s20, 1
-; SI-NEXT: v_writelane_b32 v33, s19, 2
-; SI-NEXT: s_lshr_b32 s4, s19, 16
-; SI-NEXT: v_writelane_b32 v33, s4, 3
-; SI-NEXT: v_writelane_b32 v33, s18, 4
-; SI-NEXT: v_writelane_b32 v33, s17, 5
-; SI-NEXT: v_writelane_b32 v33, s16, 6
-; SI-NEXT: v_writelane_b32 v33, s22, 7
-; SI-NEXT: v_writelane_b32 v33, s24, 8
-; SI-NEXT: s_lshr_b32 s94, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s95, s16, 16
+; SI-NEXT: v_writelane_b32 v33, s19, 0
+; SI-NEXT: v_writelane_b32 v33, s4, 1
+; SI-NEXT: s_mov_b32 s72, s18
+; SI-NEXT: v_writelane_b32 v33, s16, 2
+; SI-NEXT: s_mov_b32 s73, s20
+; SI-NEXT: v_writelane_b32 v33, s72, 3
+; SI-NEXT: v_writelane_b32 v33, s73, 4
+; SI-NEXT: v_writelane_b32 v33, s22, 5
+; SI-NEXT: s_lshr_b32 s90, s20, 16
+; SI-NEXT: s_lshr_b32 s95, s19, 16
+; SI-NEXT: s_mov_b32 s93, s17
+; SI-NEXT: s_lshr_b32 s53, s16, 16
; SI-NEXT: v_readfirstlane_b32 s89, v17
; SI-NEXT: v_readfirstlane_b32 s20, v16
; SI-NEXT: v_readfirstlane_b32 s19, v15
; SI-NEXT: v_readfirstlane_b32 s88, v14
; SI-NEXT: v_readfirstlane_b32 s79, v13
-; SI-NEXT: v_readfirstlane_b32 s18, v12
+; SI-NEXT: v_readfirstlane_b32 s51, v12
; SI-NEXT: v_readfirstlane_b32 s77, v11
; SI-NEXT: v_readfirstlane_b32 s78, v10
; SI-NEXT: v_readfirstlane_b32 s16, v9
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: v_readfirstlane_b32 s17, v7
; SI-NEXT: v_readfirstlane_b32 s82, v6
-; SI-NEXT: v_readfirstlane_b32 s39, v5
+; SI-NEXT: v_readfirstlane_b32 s80, v5
; SI-NEXT: v_readfirstlane_b32 s81, v4
-; SI-NEXT: v_readfirstlane_b32 s54, v3
-; SI-NEXT: v_readfirstlane_b32 s55, v2
-; SI-NEXT: v_readfirstlane_b32 s52, v1
-; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: v_writelane_b32 v33, s26, 9
-; SI-NEXT: s_lshr_b32 s93, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s28, 16
+; SI-NEXT: v_readfirstlane_b32 s70, v3
+; SI-NEXT: v_readfirstlane_b32 s71, v2
+; SI-NEXT: v_readfirstlane_b32 s50, v1
+; SI-NEXT: v_readfirstlane_b32 s69, v0
+; SI-NEXT: v_writelane_b32 v33, s24, 6
+; SI-NEXT: s_lshr_b32 s39, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s38, s26, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s26, 16
; SI-NEXT: s_lshr_b32 s35, s25, 16
-; SI-NEXT: s_lshr_b32 s36, s24, 16
+; SI-NEXT: s_lshr_b32 s94, s24, 16
; SI-NEXT: s_lshr_b32 s31, s23, 16
-; SI-NEXT: s_lshr_b32 s84, s22, 16
-; SI-NEXT: s_lshr_b32 s98, s89, 16
-; SI-NEXT: s_lshr_b32 s72, s20, 16
-; SI-NEXT: s_lshr_b32 s99, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s88, 16
-; SI-NEXT: s_lshr_b32 s87, s79, 16
-; SI-NEXT: s_lshr_b32 s51, s18, 16
-; SI-NEXT: s_lshr_b32 s97, s77, 16
-; SI-NEXT: s_lshr_b32 s49, s78, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
-; SI-NEXT: s_lshr_b32 s71, s76, 16
-; SI-NEXT: s_lshr_b32 s67, s17, 16
-; SI-NEXT: s_lshr_b32 s91, s82, 16
-; SI-NEXT: s_lshr_b32 s92, s39, 16
-; SI-NEXT: s_lshr_b32 s69, s81, 16
-; SI-NEXT: s_lshr_b32 s90, s54, 16
-; SI-NEXT: s_lshr_b32 s68, s55, 16
-; SI-NEXT: s_lshr_b32 s65, s52, 16
-; SI-NEXT: s_lshr_b32 s53, s70, 16
+; SI-NEXT: s_lshr_b32 s92, s22, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s65, s18, 16
+; SI-NEXT: s_lshr_b32 s99, s89, 16
+; SI-NEXT: s_lshr_b32 s74, s20, 16
+; SI-NEXT: s_lshr_b32 s98, s19, 16
+; SI-NEXT: s_lshr_b32 s18, s88, 16
+; SI-NEXT: s_lshr_b32 s97, s79, 16
+; SI-NEXT: s_lshr_b32 s49, s51, 16
+; SI-NEXT: s_lshr_b32 s96, s77, 16
+; SI-NEXT: s_lshr_b32 s91, s78, 16
+; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 s68, s76, 16
+; SI-NEXT: s_lshr_b32 s86, s17, 16
+; SI-NEXT: s_lshr_b32 s48, s82, 16
+; SI-NEXT: s_lshr_b32 s85, s80, 16
+; SI-NEXT: s_lshr_b32 s38, s81, 16
+; SI-NEXT: s_lshr_b32 s84, s70, 16
+; SI-NEXT: s_lshr_b32 s36, s71, 16
+; SI-NEXT: s_lshr_b32 s83, s50, 16
+; SI-NEXT: s_lshr_b32 s34, s69, 16
; SI-NEXT: v_readfirstlane_b32 s4, v18
-; SI-NEXT: v_writelane_b32 v33, s28, 10
+; SI-NEXT: v_writelane_b32 v33, s26, 7
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v33, s70, 11
-; SI-NEXT: v_writelane_b32 v33, s72, 12
-; SI-NEXT: s_cbranch_scc0 .LBB111_4
+; SI-NEXT: v_writelane_b32 v33, s28, 8
+; SI-NEXT: v_writelane_b32 v33, s74, 9
+; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readlane_b32 s5, v33, 5
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
+; SI-NEXT: v_readlane_b32 s7, v33, 1
+; SI-NEXT: s_and_b32 s5, s93, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: v_writelane_b32 v33, s51, 36
; SI-NEXT: s_or_b32 s61, s5, s7
-; SI-NEXT: v_readlane_b32 s5, v33, 2
-; SI-NEXT: v_readlane_b32 s7, v33, 3
+; SI-NEXT: v_readlane_b32 s5, v33, 0
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s7, s95, 16
; SI-NEXT: s_or_b32 s59, s5, s7
-; SI-NEXT: v_readlane_b32 s7, v33, 0
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 16
; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s31, 16
@@ -230326,271 +231243,363 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s7, s37, 16
; SI-NEXT: s_or_b32 s43, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s93, 16
+; SI-NEXT: s_lshl_b32 s7, s39, 16
; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s52, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s65, 16
+; SI-NEXT: s_and_b32 s5, s50, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s83, 16
; SI-NEXT: s_or_b32 s15, s5, s7
-; SI-NEXT: s_and_b32 s5, s54, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s90, 16
+; SI-NEXT: s_and_b32 s5, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s84, 16
; SI-NEXT: s_or_b32 s13, s5, s7
-; SI-NEXT: s_and_b32 s5, s39, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s92, 16
+; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s85, 16
; SI-NEXT: s_or_b32 s11, s5, s7
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s67, 16
+; SI-NEXT: s_lshl_b32 s7, s86, 16
; SI-NEXT: s_or_b32 s9, s5, s7
; SI-NEXT: s_and_b32 s5, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s83, 16
+; SI-NEXT: s_lshl_b32 s7, s87, 16
; SI-NEXT: s_or_b32 s7, s5, s7
; SI-NEXT: s_and_b32 s5, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s97, 16
+; SI-NEXT: s_lshl_b32 s62, s96, 16
; SI-NEXT: s_or_b32 s5, s5, s62
; SI-NEXT: s_and_b32 s63, s79, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s87, 16
-; SI-NEXT: s_or_b32 vcc_hi, s63, s62
+; SI-NEXT: s_lshl_b32 s62, s97, 16
+; SI-NEXT: s_lshl_b32 s60, s53, 16
+; SI-NEXT: s_mov_b32 s66, s53
+; SI-NEXT: s_or_b32 s53, s63, s62
; SI-NEXT: s_and_b32 s63, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s99, 16
-; SI-NEXT: s_mov_b32 s96, s83
-; SI-NEXT: s_mov_b32 s83, s21
-; SI-NEXT: s_mov_b32 s86, s92
-; SI-NEXT: s_mov_b32 s92, s90
-; SI-NEXT: s_mov_b32 s90, s65
-; SI-NEXT: s_or_b32 s65, s63, s62
-; SI-NEXT: s_and_b32 s21, s89, 0xffff
; SI-NEXT: s_lshl_b32 s62, s98, 16
-; SI-NEXT: s_lshl_b32 s58, s94, 16
-; SI-NEXT: s_lshl_b32 s56, s30, 16
-; SI-NEXT: s_lshl_b32 s46, s84, 16
-; SI-NEXT: s_lshl_b32 s44, s36, 16
-; SI-NEXT: s_lshl_b32 s42, s38, 16
-; SI-NEXT: s_lshl_b32 s40, s48, 16
-; SI-NEXT: s_mov_b32 s50, s48
+; SI-NEXT: s_mov_b32 s67, s21
+; SI-NEXT: s_or_b32 s55, s63, s62
+; SI-NEXT: s_and_b32 s21, s89, 0xffff
+; SI-NEXT: s_lshl_b32 s62, s99, 16
+; SI-NEXT: v_writelane_b32 v33, s23, 37
+; SI-NEXT: s_lshl_b32 s58, s65, 16
+; SI-NEXT: s_lshl_b32 s56, s90, 16
+; SI-NEXT: s_lshl_b32 s46, s92, 16
+; SI-NEXT: s_lshl_b32 s44, s94, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_lo, 16
+; SI-NEXT: s_lshl_b32 s40, s30, 16
+; SI-NEXT: s_lshl_b32 s14, s34, 16
+; SI-NEXT: s_lshl_b32 s12, s36, 16
+; SI-NEXT: s_lshl_b32 s10, s38, 16
+; SI-NEXT: s_lshl_b32 s8, s48, 16
+; SI-NEXT: s_mov_b32 s23, s48
; SI-NEXT: s_mov_b32 s48, s38
; SI-NEXT: s_mov_b32 s38, s36
-; SI-NEXT: s_mov_b32 s36, s84
-; SI-NEXT: s_mov_b32 s84, s30
-; SI-NEXT: s_mov_b32 s30, s98
-; SI-NEXT: s_mov_b32 s98, s87
+; SI-NEXT: s_mov_b32 s36, s34
+; SI-NEXT: s_mov_b32 s34, s30
+; SI-NEXT: s_mov_b32 s30, vcc_lo
+; SI-NEXT: s_mov_b32 vcc_lo, s99
+; SI-NEXT: s_mov_b32 s99, s98
+; SI-NEXT: s_mov_b32 s98, s97
+; SI-NEXT: s_mov_b32 s97, s96
+; SI-NEXT: s_mov_b32 s96, s87
; SI-NEXT: s_mov_b32 s87, s94
-; SI-NEXT: s_mov_b32 s94, s67
-; SI-NEXT: s_or_b32 s67, s21, s62
-; SI-NEXT: v_readlane_b32 s21, v33, 6
-; SI-NEXT: s_lshl_b32 s60, s95, 16
+; SI-NEXT: s_mov_b32 s94, s86
+; SI-NEXT: s_mov_b32 s86, s85
+; SI-NEXT: s_mov_b32 s85, s84
+; SI-NEXT: s_mov_b32 s84, s83
+; SI-NEXT: s_mov_b32 s83, s92
+; SI-NEXT: s_mov_b32 s92, s90
+; SI-NEXT: s_mov_b32 s90, s65
+; SI-NEXT: s_or_b32 s65, s21, s62
+; SI-NEXT: v_readlane_b32 s21, v33, 2
; SI-NEXT: s_and_b32 s21, s21, 0xffff
; SI-NEXT: s_or_b32 s62, s21, s60
; SI-NEXT: s_mov_b32 s63, s61
; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v33, s60, 13
-; SI-NEXT: v_writelane_b32 v33, s61, 14
-; SI-NEXT: v_readlane_b32 s21, v33, 4
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
+; SI-NEXT: v_writelane_b32 v33, s60, 10
+; SI-NEXT: s_and_b32 s21, s72, 0xffff
+; SI-NEXT: s_lshl_b32 s64, s74, 16
+; SI-NEXT: v_writelane_b32 v33, s61, 11
; SI-NEXT: s_or_b32 s74, s21, s58
; SI-NEXT: s_mov_b32 s75, s59
; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v33, s58, 15
-; SI-NEXT: v_writelane_b32 v33, s59, 16
-; SI-NEXT: v_readlane_b32 s21, v33, 1
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s66, s72, 16
+; SI-NEXT: v_writelane_b32 v33, s58, 12
+; SI-NEXT: s_and_b32 s21, s73, 0xffff
+; SI-NEXT: v_writelane_b32 v33, s59, 13
; SI-NEXT: s_or_b32 s72, s21, s56
; SI-NEXT: s_mov_b32 s73, s57
; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v33, s56, 17
+; SI-NEXT: v_writelane_b32 v33, s56, 14
; SI-NEXT: s_and_b32 s21, s22, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s57, 18
+; SI-NEXT: v_writelane_b32 v33, s57, 15
; SI-NEXT: s_or_b32 s60, s21, s46
; SI-NEXT: s_mov_b32 s61, s47
; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v33, s46, 19
+; SI-NEXT: v_writelane_b32 v33, s46, 16
; SI-NEXT: s_and_b32 s21, s24, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s47, 20
+; SI-NEXT: v_writelane_b32 v33, s47, 17
; SI-NEXT: s_or_b32 s58, s21, s44
; SI-NEXT: s_mov_b32 s59, s45
; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v33, s44, 21
+; SI-NEXT: v_writelane_b32 v33, s44, 18
; SI-NEXT: s_and_b32 s21, s26, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s45, 22
+; SI-NEXT: v_writelane_b32 v33, s45, 19
; SI-NEXT: s_or_b32 s56, s21, s42
; SI-NEXT: s_mov_b32 s57, s43
; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v33, s42, 23
+; SI-NEXT: v_writelane_b32 v33, s42, 20
; SI-NEXT: s_and_b32 s21, s28, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s43, 24
+; SI-NEXT: v_writelane_b32 v33, s43, 21
; SI-NEXT: s_or_b32 s46, s21, s40
; SI-NEXT: s_mov_b32 s47, s41
; SI-NEXT: s_lshr_b64 s[40:41], s[40:41], 16
-; SI-NEXT: s_lshl_b32 s14, s53, 16
-; SI-NEXT: v_writelane_b32 v33, s40, 25
-; SI-NEXT: s_and_b32 s21, s70, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s41, 26
+; SI-NEXT: v_writelane_b32 v33, s40, 22
+; SI-NEXT: s_and_b32 s21, s69, 0xffff
+; SI-NEXT: v_writelane_b32 v33, s41, 23
; SI-NEXT: s_or_b32 s44, s21, s14
; SI-NEXT: s_mov_b32 s45, s15
; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v33, s14, 27
-; SI-NEXT: s_lshl_b32 s12, s68, 16
-; SI-NEXT: v_writelane_b32 v33, s15, 28
-; SI-NEXT: s_and_b32 s14, s55, 0xffff
+; SI-NEXT: v_writelane_b32 v33, s14, 24
+; SI-NEXT: v_writelane_b32 v33, s15, 25
+; SI-NEXT: s_and_b32 s14, s71, 0xffff
; SI-NEXT: s_or_b32 s42, s14, s12
; SI-NEXT: s_mov_b32 s43, s13
; SI-NEXT: s_lshr_b64 s[12:13], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v33, s12, 29
-; SI-NEXT: s_lshl_b32 s10, s69, 16
-; SI-NEXT: v_writelane_b32 v33, s13, 30
+; SI-NEXT: v_writelane_b32 v33, s12, 26
+; SI-NEXT: v_writelane_b32 v33, s13, 27
; SI-NEXT: s_and_b32 s12, s81, 0xffff
; SI-NEXT: s_or_b32 s40, s12, s10
; SI-NEXT: s_mov_b32 s41, s11
; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v33, s10, 31
-; SI-NEXT: s_lshl_b32 s8, s91, 16
-; SI-NEXT: v_writelane_b32 v33, s11, 32
+; SI-NEXT: v_writelane_b32 v33, s10, 28
+; SI-NEXT: v_writelane_b32 v33, s11, 29
; SI-NEXT: s_and_b32 s10, s82, 0xffff
; SI-NEXT: s_or_b32 s14, s10, s8
; SI-NEXT: s_mov_b32 s15, s9
; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v33, s8, 33
-; SI-NEXT: s_lshl_b32 s6, s71, 16
-; SI-NEXT: v_writelane_b32 v33, s9, 34
+; SI-NEXT: v_writelane_b32 v33, s8, 30
+; SI-NEXT: s_lshl_b32 s6, s68, 16
+; SI-NEXT: v_writelane_b32 v33, s9, 31
; SI-NEXT: s_and_b32 s8, s76, 0xffff
; SI-NEXT: s_or_b32 s12, s8, s6
; SI-NEXT: s_mov_b32 s13, s7
; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v33, s6, 35
-; SI-NEXT: s_lshl_b32 s4, s49, 16
-; SI-NEXT: v_writelane_b32 v33, s7, 36
+; SI-NEXT: v_writelane_b32 v33, s6, 32
+; SI-NEXT: s_lshl_b32 s4, s91, 16
+; SI-NEXT: v_writelane_b32 v33, s7, 33
; SI-NEXT: s_and_b32 s6, s78, 0xffff
; SI-NEXT: s_or_b32 s10, s6, s4
; SI-NEXT: s_mov_b32 s11, s5
; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s51, 16
-; SI-NEXT: v_writelane_b32 v33, s4, 37
-; SI-NEXT: v_writelane_b32 v33, s5, 38
-; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshr_b64 s[6:7], vcc, 16
-; SI-NEXT: s_lshl_b32 s64, s85, 16
-; SI-NEXT: s_or_b32 s8, s4, vcc_lo
-; SI-NEXT: v_writelane_b32 v33, s6, 39
+; SI-NEXT: v_readlane_b32 s51, v33, 36
+; SI-NEXT: v_writelane_b32 v33, s4, 34
+; SI-NEXT: s_lshl_b32 s52, s49, 16
+; SI-NEXT: v_writelane_b32 v33, s5, 35
+; SI-NEXT: s_and_b32 s4, s51, 0xffff
+; SI-NEXT: s_lshl_b32 s54, s18, 16
+; SI-NEXT: s_or_b32 s8, s4, s52
; SI-NEXT: s_and_b32 s4, s88, 0xffff
-; SI-NEXT: v_writelane_b32 v33, s7, 40
-; SI-NEXT: s_or_b32 s6, s4, s64
+; SI-NEXT: s_or_b32 s6, s4, s54
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_mov_b32 s7, s65
+; SI-NEXT: s_mov_b32 s9, s53
+; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
+; SI-NEXT: s_or_b32 s4, s4, s64
+; SI-NEXT: s_mov_b32 s5, s65
; SI-NEXT: s_lshr_b64 s[64:65], s[64:65], 16
-; SI-NEXT: s_or_b32 s4, s4, s66
-; SI-NEXT: s_mov_b32 s5, s67
-; SI-NEXT: s_lshr_b64 s[66:67], s[66:67], 16
-; SI-NEXT: s_mov_b32 s9, vcc_hi
+; SI-NEXT: s_mov_b32 s53, s66
+; SI-NEXT: s_mov_b32 s7, s55
+; SI-NEXT: s_lshr_b64 s[54:55], s[54:55], 16
+; SI-NEXT: s_mov_b32 s21, s67
; SI-NEXT: s_mov_b32 s65, s90
; SI-NEXT: s_mov_b32 s90, s92
-; SI-NEXT: s_mov_b32 s92, s86
-; SI-NEXT: s_mov_b32 s21, s83
-; SI-NEXT: s_mov_b32 s83, s96
-; SI-NEXT: s_mov_b32 s67, s94
+; SI-NEXT: s_mov_b32 s92, s83
+; SI-NEXT: s_mov_b32 s83, s84
+; SI-NEXT: s_mov_b32 s84, s85
+; SI-NEXT: s_mov_b32 s85, s86
+; SI-NEXT: s_mov_b32 s86, s94
; SI-NEXT: s_mov_b32 s94, s87
-; SI-NEXT: s_mov_b32 s87, s98
-; SI-NEXT: s_mov_b32 s98, s30
-; SI-NEXT: s_mov_b32 s30, s84
-; SI-NEXT: s_mov_b32 s84, s36
+; SI-NEXT: s_mov_b32 s87, s96
+; SI-NEXT: s_mov_b32 s96, s97
+; SI-NEXT: s_mov_b32 s97, s98
+; SI-NEXT: s_mov_b32 s98, s99
+; SI-NEXT: s_mov_b32 s99, vcc_lo
+; SI-NEXT: s_mov_b32 vcc_lo, s30
+; SI-NEXT: s_mov_b32 s30, s34
+; SI-NEXT: s_mov_b32 s34, s36
; SI-NEXT: s_mov_b32 s36, s38
; SI-NEXT: s_mov_b32 s38, s48
-; SI-NEXT: s_mov_b32 s48, s50
-; SI-NEXT: s_cbranch_execnz .LBB111_3
-; SI-NEXT: .LBB111_2: ; %cmp.true
+; SI-NEXT: s_mov_b32 s48, s23
+; SI-NEXT: v_readlane_b32 s23, v33, 37
+; SI-NEXT: s_mov_b64 s[66:67], 0
+; SI-NEXT: s_branch .LBB111_3
+; SI-NEXT: .LBB111_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b64 s[66:67], -1
+; SI-NEXT: v_writelane_b32 v33, s4, 10
+; SI-NEXT: v_writelane_b32 v33, s5, 11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: v_writelane_b32 v33, s4, 12
+; SI-NEXT: v_writelane_b32 v33, s5, 13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 14
+; SI-NEXT: v_writelane_b32 v33, s5, 15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 16
+; SI-NEXT: v_writelane_b32 v33, s5, 17
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 18
+; SI-NEXT: v_writelane_b32 v33, s5, 19
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 20
+; SI-NEXT: v_writelane_b32 v33, s5, 21
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 22
+; SI-NEXT: v_writelane_b32 v33, s5, 23
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 24
+; SI-NEXT: v_writelane_b32 v33, s5, 25
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 26
+; SI-NEXT: v_writelane_b32 v33, s5, 27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 28
+; SI-NEXT: v_writelane_b32 v33, s5, 29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 30
+; SI-NEXT: v_writelane_b32 v33, s5, 31
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 32
+; SI-NEXT: v_writelane_b32 v33, s5, 33
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v33, s4, 34
+; SI-NEXT: v_writelane_b32 v33, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB111_3: ; %Flow
+; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
+; SI-NEXT: v_readlane_b32 s66, v33, 10
+; SI-NEXT: v_readlane_b32 s67, v33, 11
+; SI-NEXT: s_mov_b32 s22, s66
+; SI-NEXT: v_readlane_b32 s66, v33, 12
+; SI-NEXT: v_readlane_b32 s67, v33, 13
+; SI-NEXT: s_mov_b32 s24, s66
+; SI-NEXT: v_readlane_b32 s66, v33, 14
+; SI-NEXT: v_readlane_b32 s67, v33, 15
+; SI-NEXT: s_mov_b32 s26, s66
+; SI-NEXT: v_readlane_b32 s66, v33, 16
+; SI-NEXT: s_cselect_b32 s55, 1, 0
+; SI-NEXT: v_readlane_b32 s67, v33, 17
+; SI-NEXT: s_cmp_lg_u32 s55, 1
+; SI-NEXT: s_mov_b32 s28, s66
+; SI-NEXT: v_readlane_b32 s66, v33, 18
+; SI-NEXT: v_readlane_b32 s67, v33, 19
+; SI-NEXT: s_cbranch_scc1 .LBB111_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: v_readlane_b32 s5, v33, 12
+; SI-NEXT: v_readlane_b32 s5, v33, 9
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_add_i32 s89, s89, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s89, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s98, 16
+; SI-NEXT: s_lshl_b32 s6, s99, 16
; SI-NEXT: s_add_i32 s88, s88, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s88, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s85, 16
+; SI-NEXT: s_lshl_b32 s7, s18, 16
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s99, 16
-; SI-NEXT: s_add_i32 s18, s18, 3
+; SI-NEXT: s_lshl_b32 s8, s98, 16
+; SI-NEXT: s_add_i32 s18, s51, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s51, 16
+; SI-NEXT: s_lshl_b32 s9, s49, 16
; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s79, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s87, 16
+; SI-NEXT: s_lshl_b32 s10, s97, 16
; SI-NEXT: s_add_i32 s78, s78, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s78, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s49, 16
+; SI-NEXT: s_lshl_b32 s11, s91, 16
; SI-NEXT: s_add_i32 s77, s77, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s77, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s97, 16
+; SI-NEXT: s_lshl_b32 s12, s96, 16
; SI-NEXT: s_add_i32 s76, s76, 3
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_and_b32 s12, s76, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s71, 16
+; SI-NEXT: s_lshl_b32 s13, s68, 16
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_and_b32 s13, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s83, 16
+; SI-NEXT: s_lshl_b32 s14, s87, 16
; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: s_and_b32 s14, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s91, 16
+; SI-NEXT: s_lshl_b32 s15, s48, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s15, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s67, 16
+; SI-NEXT: s_lshl_b32 s16, s86, 16
; SI-NEXT: s_add_i32 s81, s81, 3
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s16, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s69, 16
+; SI-NEXT: s_lshl_b32 s17, s38, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s80, s39, 3
+; SI-NEXT: s_add_i32 s80, s80, 3
; SI-NEXT: s_add_i32 s40, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s92, 16
+; SI-NEXT: s_lshl_b32 s17, s85, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s71, s55, 3
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s41, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s68, 16
+; SI-NEXT: s_lshl_b32 s17, s36, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s70, s54, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s42, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s90, 16
+; SI-NEXT: s_lshl_b32 s17, s84, 16
; SI-NEXT: s_or_b32 s16, s17, s16
+; SI-NEXT: s_add_i32 s69, s69, 3
; SI-NEXT: s_add_i32 s43, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 11
-; SI-NEXT: s_add_i32 s69, s16, 3
; SI-NEXT: s_and_b32 s16, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s53, 16
+; SI-NEXT: s_lshl_b32 s17, s34, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_add_i32 s68, s52, 3
+; SI-NEXT: s_add_i32 s68, s50, 3
; SI-NEXT: s_add_i32 s44, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s65, 16
+; SI-NEXT: s_lshl_b32 s17, s83, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s45, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 10
+; SI-NEXT: v_readlane_b32 s16, v33, 8
; SI-NEXT: s_add_i32 s28, s16, 3
; SI-NEXT: s_and_b32 s16, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s48, 16
+; SI-NEXT: s_lshl_b32 s17, s30, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s46, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s93, 16
+; SI-NEXT: s_lshl_b32 s17, s39, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s47, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 9
+; SI-NEXT: v_readlane_b32 s16, v33, 7
; SI-NEXT: s_add_i32 s26, s16, 3
; SI-NEXT: s_and_b32 s16, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s38, 16
+; SI-NEXT: s_lshl_b32 s17, vcc_lo, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s56, s16, 0x30000
@@ -230598,10 +231607,10 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s17, s37, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s57, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 8
+; SI-NEXT: v_readlane_b32 s16, v33, 6
; SI-NEXT: s_add_i32 s24, s16, 3
; SI-NEXT: s_and_b32 s16, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s36, 16
+; SI-NEXT: s_lshl_b32 s17, s94, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s58, s16, 0x30000
@@ -230609,10 +231618,10 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s17, s35, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s59, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 7
+; SI-NEXT: v_readlane_b32 s16, v33, 5
; SI-NEXT: s_add_i32 s22, s16, 3
; SI-NEXT: s_and_b32 s16, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s84, 16
+; SI-NEXT: s_lshl_b32 s17, s92, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s60, s16, 0x30000
@@ -230620,245 +231629,217 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s17, s31, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s61, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 1
+; SI-NEXT: v_readlane_b32 s16, v33, 4
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s30, 16
+; SI-NEXT: s_lshl_b32 s17, s90, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s72, s16, 0x30000
; SI-NEXT: s_add_i32 s16, s21, 3
-; SI-NEXT: v_readlane_b32 s17, v33, 0
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s73, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 4
+; SI-NEXT: v_readlane_b32 s16, v33, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s94, 16
+; SI-NEXT: s_lshl_b32 s17, s65, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s74, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 2
+; SI-NEXT: v_readlane_b32 s16, v33, 0
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v33, 3
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s17, s95, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s75, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 6
+; SI-NEXT: v_readlane_b32 s16, v33, 2
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s95, 16
+; SI-NEXT: s_lshl_b32 s17, s53, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s62, s16, 0x30000
-; SI-NEXT: v_readlane_b32 s16, v33, 5
-; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s16, s93, 3
+; SI-NEXT: v_readlane_b32 s17, v33, 1
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s34, 16
+; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s63, s16, 0x30000
-; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 13
-; SI-NEXT: v_writelane_b32 v33, s17, 14
-; SI-NEXT: s_lshr_b64 s[16:17], s[74:75], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 15
-; SI-NEXT: v_writelane_b32 v33, s17, 16
-; SI-NEXT: s_lshr_b64 s[16:17], s[72:73], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 17
-; SI-NEXT: v_writelane_b32 v33, s17, 18
-; SI-NEXT: s_lshr_b64 s[16:17], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 19
-; SI-NEXT: v_writelane_b32 v33, s17, 20
-; SI-NEXT: s_lshr_b64 s[16:17], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 21
-; SI-NEXT: v_writelane_b32 v33, s17, 22
; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 23
-; SI-NEXT: v_writelane_b32 v33, s17, 24
+; SI-NEXT: v_writelane_b32 v33, s16, 20
+; SI-NEXT: v_writelane_b32 v33, s17, 21
; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 25
-; SI-NEXT: v_writelane_b32 v33, s17, 26
+; SI-NEXT: v_writelane_b32 v33, s16, 22
+; SI-NEXT: v_writelane_b32 v33, s17, 23
; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 27
-; SI-NEXT: v_writelane_b32 v33, s17, 28
+; SI-NEXT: v_writelane_b32 v33, s16, 24
+; SI-NEXT: v_writelane_b32 v33, s17, 25
; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v33, s16, 29
-; SI-NEXT: v_writelane_b32 v33, s17, 30
+; SI-NEXT: v_writelane_b32 v33, s16, 26
+; SI-NEXT: v_writelane_b32 v33, s17, 27
; SI-NEXT: s_lshr_b64 s[16:17], s[40:41], 16
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: v_writelane_b32 v33, s16, 31
-; SI-NEXT: v_writelane_b32 v33, s17, 32
+; SI-NEXT: v_writelane_b32 v33, s16, 28
+; SI-NEXT: v_writelane_b32 v33, s17, 29
; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 16
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: v_writelane_b32 v33, s16, 33
-; SI-NEXT: v_writelane_b32 v33, s17, 34
+; SI-NEXT: v_writelane_b32 v33, s16, 30
+; SI-NEXT: v_writelane_b32 v33, s17, 31
; SI-NEXT: s_lshr_b64 s[16:17], s[12:13], 16
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: v_writelane_b32 v33, s16, 35
-; SI-NEXT: v_writelane_b32 v33, s17, 36
+; SI-NEXT: v_writelane_b32 v33, s16, 32
+; SI-NEXT: v_writelane_b32 v33, s17, 33
; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: s_add_i32 s8, s8, 0x30000
-; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: v_writelane_b32 v33, s16, 37
-; SI-NEXT: v_writelane_b32 v33, s17, 38
-; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: v_writelane_b32 v33, s16, 39
-; SI-NEXT: v_writelane_b32 v33, s17, 40
-; SI-NEXT: s_lshr_b64 s[64:65], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[66:67], s[4:5], 16
-; SI-NEXT: s_lshr_b32 s16, s75, 16
-; SI-NEXT: s_lshr_b32 s34, s63, 16
-; SI-NEXT: v_writelane_b32 v33, s16, 3
-; SI-NEXT: s_lshr_b32 s16, s73, 16
+; SI-NEXT: s_add_i32 s8, s8, 0x30000
+; SI-NEXT: s_add_i32 s9, s9, 0x30000
+; SI-NEXT: v_writelane_b32 v33, s16, 34
+; SI-NEXT: s_lshr_b64 s[22:23], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[26:27], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[28:29], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[66:67], s[58:59], 16
+; SI-NEXT: v_writelane_b32 v33, s17, 35
+; SI-NEXT: s_lshr_b64 s[52:53], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[54:55], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[64:65], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s16, s63, 16
+; SI-NEXT: s_lshr_b32 s95, s75, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s73, 16
; SI-NEXT: s_lshr_b32 s31, s61, 16
; SI-NEXT: s_lshr_b32 s35, s59, 16
; SI-NEXT: s_lshr_b32 s37, s57, 16
-; SI-NEXT: s_lshr_b32 s93, s47, 16
-; SI-NEXT: s_lshr_b32 s65, s45, 16
-; SI-NEXT: s_lshr_b32 s90, s43, 16
-; SI-NEXT: s_lshr_b32 s92, s41, 16
-; SI-NEXT: s_lshr_b32 s67, s15, 16
-; SI-NEXT: s_lshr_b32 s83, s13, 16
-; SI-NEXT: s_lshr_b32 s97, s11, 16
-; SI-NEXT: s_lshr_b32 s87, s9, 16
-; SI-NEXT: s_lshr_b32 s99, s7, 16
-; SI-NEXT: s_lshr_b32 s98, s5, 16
-; SI-NEXT: v_writelane_b32 v33, s16, 0
-; SI-NEXT: .LBB111_3: ; %end
-; SI-NEXT: v_readlane_b32 s18, v33, 13
+; SI-NEXT: s_lshr_b32 s39, s47, 16
+; SI-NEXT: s_lshr_b32 s83, s45, 16
+; SI-NEXT: s_lshr_b32 s84, s43, 16
+; SI-NEXT: s_lshr_b32 s85, s41, 16
+; SI-NEXT: s_lshr_b32 s86, s15, 16
+; SI-NEXT: s_lshr_b32 s87, s13, 16
+; SI-NEXT: s_lshr_b32 s96, s11, 16
+; SI-NEXT: s_lshr_b32 s97, s9, 16
+; SI-NEXT: s_lshr_b32 s98, s7, 16
+; SI-NEXT: s_lshr_b32 s99, s5, 16
+; SI-NEXT: v_writelane_b32 v33, s16, 1
+; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: s_and_b32 s16, s62, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s18, 16
-; SI-NEXT: v_readlane_b32 s19, v33, 14
+; SI-NEXT: s_lshl_b32 s17, s22, 16
+; SI-NEXT: v_readlane_b32 s18, v33, 1
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s63, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s34, 16
-; SI-NEXT: v_readlane_b32 s20, v33, 15
+; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s74, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s20, 16
-; SI-NEXT: v_readlane_b32 s20, v33, 3
-; SI-NEXT: v_readlane_b32 s21, v33, 16
+; SI-NEXT: s_lshl_b32 s19, s24, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s20, 16
-; SI-NEXT: v_readlane_b32 s22, v33, 17
+; SI-NEXT: s_lshl_b32 s20, s95, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s72, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s22, 16
-; SI-NEXT: v_readlane_b32 s22, v33, 0
-; SI-NEXT: v_readlane_b32 s23, v33, 18
+; SI-NEXT: s_lshl_b32 s21, s26, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: v_readlane_b32 s24, v33, 19
+; SI-NEXT: s_lshl_b32 s22, vcc_hi, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s60, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s24, 16
-; SI-NEXT: v_readlane_b32 s25, v33, 20
+; SI-NEXT: s_lshl_b32 s23, s28, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s61, 0xffff
; SI-NEXT: s_lshl_b32 s24, s31, 16
-; SI-NEXT: v_readlane_b32 s26, v33, 21
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s58, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s26, 16
-; SI-NEXT: v_readlane_b32 s27, v33, 22
+; SI-NEXT: s_lshl_b32 s25, s66, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s59, 0xffff
; SI-NEXT: s_lshl_b32 s26, s35, 16
-; SI-NEXT: v_readlane_b32 s28, v33, 23
+; SI-NEXT: v_readlane_b32 s28, v33, 20
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s56, 0xffff
; SI-NEXT: s_lshl_b32 s27, s28, 16
-; SI-NEXT: v_readlane_b32 s29, v33, 24
+; SI-NEXT: v_readlane_b32 s29, v33, 21
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s57, 0xffff
; SI-NEXT: s_lshl_b32 s28, s37, 16
-; SI-NEXT: v_readlane_b32 s56, v33, 25
+; SI-NEXT: v_readlane_b32 s56, v33, 22
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s46, 0xffff
; SI-NEXT: s_lshl_b32 s29, s56, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s93, 16
+; SI-NEXT: s_lshl_b32 s46, s39, 16
; SI-NEXT: s_or_b32 s29, s29, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 27
+; SI-NEXT: v_readlane_b32 s46, v33, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 28
+; SI-NEXT: v_readlane_b32 s47, v33, 25
; SI-NEXT: s_or_b32 s44, s44, s46
; SI-NEXT: s_and_b32 s45, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s65, 16
+; SI-NEXT: s_lshl_b32 s46, s83, 16
; SI-NEXT: s_or_b32 s45, s45, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 29
+; SI-NEXT: v_readlane_b32 s46, v33, 26
; SI-NEXT: s_and_b32 s42, s42, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 30
+; SI-NEXT: v_readlane_b32 s47, v33, 27
; SI-NEXT: s_or_b32 s42, s42, s46
; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s90, 16
+; SI-NEXT: s_lshl_b32 s46, s84, 16
; SI-NEXT: s_or_b32 s43, s43, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 31
+; SI-NEXT: v_readlane_b32 s46, v33, 28
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 32
+; SI-NEXT: v_readlane_b32 s47, v33, 29
; SI-NEXT: s_or_b32 s40, s40, s46
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s92, 16
+; SI-NEXT: s_lshl_b32 s46, s85, 16
; SI-NEXT: s_or_b32 s41, s41, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 33
+; SI-NEXT: v_readlane_b32 s46, v33, 30
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 34
+; SI-NEXT: v_readlane_b32 s47, v33, 31
; SI-NEXT: s_or_b32 s14, s14, s46
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s67, 16
+; SI-NEXT: s_lshl_b32 s46, s86, 16
; SI-NEXT: s_or_b32 s15, s15, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 35
+; SI-NEXT: v_readlane_b32 s46, v33, 32
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 36
+; SI-NEXT: v_readlane_b32 s47, v33, 33
; SI-NEXT: s_or_b32 s12, s12, s46
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s83, 16
+; SI-NEXT: s_lshl_b32 s46, s87, 16
; SI-NEXT: s_or_b32 s13, s13, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 37
+; SI-NEXT: v_readlane_b32 s46, v33, 34
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 38
; SI-NEXT: s_or_b32 s10, s10, s46
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s97, 16
+; SI-NEXT: s_lshl_b32 s46, s96, 16
; SI-NEXT: s_or_b32 s11, s11, s46
-; SI-NEXT: v_readlane_b32 s46, v33, 39
; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_lshl_b32 s46, s52, 16
; SI-NEXT: s_or_b32 s8, s8, s46
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s87, 16
+; SI-NEXT: s_lshl_b32 s46, s97, 16
; SI-NEXT: s_or_b32 s9, s9, s46
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s64, 16
+; SI-NEXT: s_lshl_b32 s46, s54, 16
; SI-NEXT: s_or_b32 s6, s6, s46
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s99, 16
+; SI-NEXT: s_lshl_b32 s46, s98, 16
; SI-NEXT: s_or_b32 s7, s7, s46
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s66, 16
+; SI-NEXT: s_lshl_b32 s46, s64, 16
; SI-NEXT: s_or_b32 s4, s4, s46
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s98, 16
+; SI-NEXT: s_lshl_b32 s46, s99, 16
; SI-NEXT: s_or_b32 s5, s5, s46
; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s57, v33, 26
-; SI-NEXT: v_readlane_b32 s47, v33, 40
+; SI-NEXT: v_readlane_b32 s57, v33, 23
+; SI-NEXT: v_readlane_b32 s47, v33, 35
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -230932,68 +231913,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB111_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 13
-; SI-NEXT: v_writelane_b32 v33, s5, 14
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: v_writelane_b32 v33, s4, 15
-; SI-NEXT: v_writelane_b32 v33, s5, 16
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 17
-; SI-NEXT: v_writelane_b32 v33, s5, 18
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 19
-; SI-NEXT: v_writelane_b32 v33, s5, 20
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 21
-; SI-NEXT: v_writelane_b32 v33, s5, 22
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 23
-; SI-NEXT: v_writelane_b32 v33, s5, 24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 25
-; SI-NEXT: v_writelane_b32 v33, s5, 26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 27
-; SI-NEXT: v_writelane_b32 v33, s5, 28
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 29
-; SI-NEXT: v_writelane_b32 v33, s5, 30
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 31
-; SI-NEXT: v_writelane_b32 v33, s5, 32
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 33
-; SI-NEXT: v_writelane_b32 v33, s5, 34
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 35
-; SI-NEXT: v_writelane_b32 v33, s5, 36
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 37
-; SI-NEXT: v_writelane_b32 v33, s5, 38
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v33, s4, 39
-; SI-NEXT: v_writelane_b32 v33, s5, 40
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB111_2
;
; VI-LABEL: bitcast_v64i16_to_v64f16_scalar:
; VI: ; %bb.0:
@@ -231029,10 +231948,18 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB111_4
+; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB111_3
-; VI-NEXT: .LBB111_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB111_3
+; VI-NEXT: .LBB111_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB111_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB111_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s47, 0xffff0000
; VI-NEXT: s_add_i32 s5, s47, 3
; VI-NEXT: s_and_b32 s47, s46, 0xffff0000
@@ -231193,7 +232120,7 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s45, s45, 0x30000
; VI-NEXT: s_add_i32 s46, s46, 0x30000
; VI-NEXT: s_add_i32 s47, s4, 0x30000
-; VI-NEXT: .LBB111_3: ; %end
+; VI-NEXT: .LBB111_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 6
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
@@ -231239,8 +232166,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB111_4:
-; VI-NEXT: s_branch .LBB111_2
;
; GFX9-LABEL: bitcast_v64i16_to_v64f16_scalar:
; GFX9: ; %bb.0:
@@ -231278,10 +232203,18 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB111_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB111_4
-; GFX9-NEXT: .LBB111_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB111_3
+; GFX9-NEXT: .LBB111_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB111_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -231314,10 +232247,8 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v18, s38, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s36, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB111_5
-; GFX9-NEXT: .LBB111_3:
-; GFX9-NEXT: s_branch .LBB111_2
-; GFX9-NEXT: .LBB111_4:
+; GFX9-NEXT: s_branch .LBB111_6
+; GFX9-NEXT: .LBB111_5:
; GFX9-NEXT: v_mov_b32_e32 v16, s36
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v17, s37
@@ -231350,7 +232281,7 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB111_5: ; %end
+; GFX9-NEXT: .LBB111_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v32, 8
; GFX9-NEXT: v_readlane_b32 s31, v32, 9
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -231404,11 +232335,16 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s13, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB111_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB111_4
-; GFX11-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB111_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -231442,8 +232378,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v17, s37, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s36, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_branch .LBB111_5
-; GFX11-NEXT: .LBB111_3:
-; GFX11-NEXT: s_branch .LBB111_2
; GFX11-NEXT: .LBB111_4:
; GFX11-NEXT: v_dual_mov_b32 v16, s36 :: v_dual_mov_b32 v17, s37
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 7e231f48de12a..c8cef65fd3625 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -93,86 +93,107 @@ define inreg <4 x float> @bitcast_v4i32_to_v4f32_scalar(<4 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v4i32_to_v4f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v4i32_to_v4f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v4i32_to_v4f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -275,18 +296,24 @@ define inreg <4 x i32> @bitcast_v4f32_to_v4i32_scalar(<4 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -297,18 +324,24 @@ define inreg <4 x i32> @bitcast_v4f32_to_v4i32_scalar(<4 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -319,18 +352,24 @@ define inreg <4 x i32> @bitcast_v4f32_to_v4i32_scalar(<4 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -342,18 +381,21 @@ define inreg <4 x i32> @bitcast_v4f32_to_v4i32_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -462,86 +504,107 @@ define inreg <2 x i64> @bitcast_v4i32_to_v2i64_scalar(<4 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v4i32_to_v2i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v4i32_to_v2i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v4i32_to_v2i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -647,86 +710,107 @@ define inreg <4 x i32> @bitcast_v2i64_to_v4i32_scalar(<2 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v2i64_to_v4i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v2i64_to_v4i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v2i64_to_v4i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -831,86 +915,107 @@ define inreg <2 x double> @bitcast_v4i32_to_v2f64_scalar(<4 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v4i32_to_v2f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v4i32_to_v2f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v4i32_to_v2f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1010,16 +1115,22 @@ define inreg <4 x i32> @bitcast_v2f64_to_v4i32_scalar(<2 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1030,16 +1141,22 @@ define inreg <4 x i32> @bitcast_v2f64_to_v4i32_scalar(<2 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1050,16 +1167,22 @@ define inreg <4 x i32> @bitcast_v2f64_to_v4i32_scalar(<2 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1071,16 +1194,19 @@ define inreg <4 x i32> @bitcast_v2f64_to_v4i32_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1215,14 +1341,26 @@ define inreg <8 x i16> @bitcast_v4i32_to_v8i16_scalar(<4 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1231,7 +1369,7 @@ define inreg <8 x i16> @bitcast_v4i32_to_v8i16_scalar(<4 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -1249,76 +1387,85 @@ define inreg <8 x i16> @bitcast_v4i32_to_v8i16_scalar(<4 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v4i32_to_v8i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v4i32_to_v8i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v4i32_to_v8i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1486,7 +1633,7 @@ define inreg <4 x i32> @bitcast_v8i16_to_v4i32_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -1500,8 +1647,17 @@ define inreg <4 x i32> @bitcast_v8i16_to_v4i32_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -1522,24 +1678,29 @@ define inreg <4 x i32> @bitcast_v8i16_to_v4i32_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v8i16_to_v4i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s19, 3
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1560,31 +1721,35 @@ define inreg <4 x i32> @bitcast_v8i16_to_v4i32_scalar(<8 x i16> inreg %a, i32 in
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v8i16_to_v4i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1596,18 +1761,21 @@ define inreg <4 x i32> @bitcast_v8i16_to_v4i32_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1742,14 +1910,26 @@ define inreg <8 x half> @bitcast_v4i32_to_v8f16_scalar(<4 x i32> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1758,7 +1938,7 @@ define inreg <8 x half> @bitcast_v4i32_to_v8f16_scalar(<4 x i32> inreg %a, i32 i
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -1776,76 +1956,85 @@ define inreg <8 x half> @bitcast_v4i32_to_v8f16_scalar(<4 x i32> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v4i32_to_v8f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v4i32_to_v8f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v4i32_to_v8f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2030,7 +2219,7 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -2044,8 +2233,17 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -2079,10 +2277,7 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2093,10 +2288,18 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2119,9 +2322,7 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2132,19 +2333,25 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2156,18 +2363,21 @@ define inreg <4 x i32> @bitcast_v8f16_to_v4i32_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2323,7 +2533,7 @@ define inreg <8 x bfloat> @bitcast_v4i32_to_v8bf16_scalar(<4 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s19, 16
@@ -2333,8 +2543,24 @@ define inreg <8 x bfloat> @bitcast_v4i32_to_v8bf16_scalar(<4 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s10, s17, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s12, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -2347,7 +2573,7 @@ define inreg <8 x bfloat> @bitcast_v4i32_to_v8bf16_scalar(<4 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s10, s17, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s12, s16, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
@@ -2365,80 +2591,85 @@ define inreg <8 x bfloat> @bitcast_v4i32_to_v8bf16_scalar(<4 x i32> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v4i32_to_v8bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v4i32_to_v8bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_v4i32_to_v8bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2903,7 +3134,7 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
@@ -2913,8 +3144,17 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -2939,20 +3179,25 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v8bf16_to_v4i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -3029,9 +3274,7 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v1, v5
; VI-NEXT: v_mov_b32_e32 v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3042,10 +3285,18 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -3125,9 +3376,7 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3139,11 +3388,16 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-TRUE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -3225,8 +3479,6 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB23_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB23_2
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3237,11 +3489,16 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-FAKE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 16
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s3
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -3328,8 +3585,6 @@ define inreg <4 x i32> @bitcast_v8bf16_to_v4i32_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v7, 16, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB23_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB23_2
; GFX11-FAKE16-NEXT: .LBB23_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3682,7 +3937,7 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_4
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s22, s19, 24
; SI-NEXT: s_lshr_b32 s23, s19, 16
@@ -3696,8 +3951,28 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB25_3
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3714,7 +3989,7 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s25, s17, 24
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s17, 8
-; SI-NEXT: .LBB25_3: ; %end
+; SI-NEXT: .LBB25_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_mov_b32_e32 v2, s12
@@ -3732,26 +4007,12 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v14, s23
; SI-NEXT: v_mov_b32_e32 v15, s22
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: s_branch .LBB25_2
;
; VI-LABEL: bitcast_v4i32_to_v16i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_4
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s19, 24
; VI-NEXT: s_lshr_b32 s11, s19, 16
@@ -3765,8 +4026,28 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s23, s16, 8
; VI-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB25_3
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s19, s19, 3
@@ -3783,7 +4064,7 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s21, s17, 8
; VI-NEXT: s_lshr_b32 s22, s16, 16
; VI-NEXT: s_lshr_b32 s23, s16, 8
-; VI-NEXT: .LBB25_3: ; %end
+; VI-NEXT: .LBB25_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s23
; VI-NEXT: v_mov_b32_e32 v2, s22
@@ -3801,26 +4082,12 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v14, s11
; VI-NEXT: v_mov_b32_e32 v15, s10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_4:
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB25_2
;
; GFX9-LABEL: bitcast_v4i32_to_v16i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s19, 24
; GFX9-NEXT: s_lshr_b32 s11, s19, 16
@@ -3834,8 +4101,28 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
; GFX9-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB25_3
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
@@ -3852,7 +4139,7 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s21, s17, 8
; GFX9-NEXT: s_lshr_b32 s22, s16, 16
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
-; GFX9-NEXT: .LBB25_3: ; %end
+; GFX9-NEXT: .LBB25_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s23
; GFX9-NEXT: v_mov_b32_e32 v2, s22
@@ -3870,27 +4157,13 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v14, s11
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_4:
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB25_2
;
; GFX11-LABEL: bitcast_v4i32_to_v16i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s18, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s3, 24
; GFX11-NEXT: s_lshr_b32 s9, s3, 16
@@ -3904,9 +4177,28 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s18
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_3
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB25_3
+; GFX11-NEXT: .LBB25_2:
+; GFX11-NEXT: s_mov_b32 s18, -1
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB25_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -3923,7 +4215,7 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s15, s1, 8
; GFX11-NEXT: s_lshr_b32 s16, s0, 16
; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: .LBB25_3: ; %end
+; GFX11-NEXT: .LBB25_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
; GFX11-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
@@ -3934,20 +4226,6 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
; GFX11-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_4:
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB25_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4498,7 +4776,7 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4536,8 +4814,17 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4590,15 +4877,12 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v16i8_to_v4i32_scalar:
; VI: ; %bb.0:
@@ -4607,7 +4891,7 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -4634,8 +4918,17 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -4677,11 +4970,8 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v16i8_to_v4i32_scalar:
; GFX9: ; %bb.0:
@@ -4690,7 +4980,7 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -4717,8 +5007,17 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB27_3
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -4757,18 +5056,15 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB27_3: ; %end
+; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB27_2
;
; GFX11-LABEL: bitcast_v16i8_to_v4i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4790,9 +5086,17 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB27_3
+; GFX11-NEXT: .LBB27_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB27_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -4831,11 +5135,8 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB27_3: ; %end
+; GFX11-NEXT: .LBB27_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB27_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4938,18 +5239,24 @@ define inreg <2 x i64> @bitcast_v4f32_to_v2i64_scalar(<4 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4960,18 +5267,24 @@ define inreg <2 x i64> @bitcast_v4f32_to_v2i64_scalar(<4 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4982,18 +5295,24 @@ define inreg <2 x i64> @bitcast_v4f32_to_v2i64_scalar(<4 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5005,18 +5324,21 @@ define inreg <2 x i64> @bitcast_v4f32_to_v2i64_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5126,86 +5448,107 @@ define inreg <4 x float> @bitcast_v2i64_to_v4f32_scalar(<2 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v2i64_to_v4f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v2i64_to_v4f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_3
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB31_3: ; %end
+; GFX9-NEXT: .LBB31_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_4:
-; GFX9-NEXT: s_branch .LBB31_2
;
; GFX11-LABEL: bitcast_v2i64_to_v4f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_3
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB31_3: ; %end
+; GFX11-NEXT: .LBB31_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_4:
-; GFX11-NEXT: s_branch .LBB31_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5308,18 +5651,24 @@ define inreg <2 x double> @bitcast_v4f32_to_v2f64_scalar(<4 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5330,18 +5679,24 @@ define inreg <2 x double> @bitcast_v4f32_to_v2f64_scalar(<4 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5352,18 +5707,24 @@ define inreg <2 x double> @bitcast_v4f32_to_v2f64_scalar(<4 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5375,18 +5736,21 @@ define inreg <2 x double> @bitcast_v4f32_to_v2f64_scalar(<4 x float> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5490,16 +5854,22 @@ define inreg <4 x float> @bitcast_v2f64_to_v4f32_scalar(<2 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5510,16 +5880,22 @@ define inreg <4 x float> @bitcast_v2f64_to_v4f32_scalar(<2 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5530,16 +5906,22 @@ define inreg <4 x float> @bitcast_v2f64_to_v4f32_scalar(<2 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5551,16 +5933,19 @@ define inreg <4 x float> @bitcast_v2f64_to_v4f32_scalar(<2 x double> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5693,14 +6078,26 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s11, s19, 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
@@ -5709,14 +6106,8 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; SI-NEXT: s_branch .LBB37_5
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: s_branch .LBB37_6
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5725,7 +6116,7 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s11
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s6
-; SI-NEXT: .LBB37_5: ; %end
+; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -5744,18 +6135,24 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5766,18 +6163,24 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5789,18 +6192,21 @@ define inreg <8 x i16> @bitcast_v4f32_to_v8i16_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5972,7 +6378,7 @@ define inreg <4 x float> @bitcast_v8i16_to_v4f32_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -5986,8 +6392,17 @@ define inreg <4 x float> @bitcast_v8i16_to_v4f32_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -6008,24 +6423,29 @@ define inreg <4 x float> @bitcast_v8i16_to_v4f32_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v8i16_to_v4f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s19, 3
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -6046,31 +6466,35 @@ define inreg <4 x float> @bitcast_v8i16_to_v4f32_scalar(<8 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v8i16_to_v4f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6082,18 +6506,21 @@ define inreg <4 x float> @bitcast_v8i16_to_v4f32_scalar(<8 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6226,14 +6653,26 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s11, s19, 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
@@ -6242,14 +6681,8 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6258,7 +6691,7 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s11
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s6
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -6277,18 +6710,24 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6299,18 +6738,24 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6322,18 +6767,21 @@ define inreg <8 x half> @bitcast_v4f32_to_v8f16_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6522,7 +6970,7 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_3
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -6536,8 +6984,17 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB43_4
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -6571,10 +7028,7 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB43_2
-; SI-NEXT: .LBB43_4:
+; SI-NEXT: .LBB43_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -6585,10 +7039,18 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -6611,9 +7073,7 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6624,19 +7084,25 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6648,18 +7114,21 @@ define inreg <4 x float> @bitcast_v8f16_to_v4f32_scalar(<8 x half> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6813,7 +7282,7 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s19, 16
@@ -6823,8 +7292,24 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; SI-NEXT: s_lshl_b32 s11, s17, 16
; SI-NEXT: s_and_b32 s12, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s13, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
@@ -6837,18 +7322,8 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
; SI-NEXT: v_mov_b32_e32 v0, s13
; SI-NEXT: v_mov_b32_e32 v1, s12
; SI-NEXT: v_mov_b32_e32 v2, s11
@@ -6857,7 +7332,7 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v6, s8
; SI-NEXT: v_mov_b32_e32 v4, s7
; SI-NEXT: v_mov_b32_e32 v5, s6
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6880,18 +7355,24 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6902,18 +7383,24 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6925,18 +7412,21 @@ define inreg <8 x bfloat> @bitcast_v4f32_to_v8bf16_scalar(<4 x float> inreg %a,
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7405,7 +7895,7 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB47_4
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
@@ -7415,8 +7905,17 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_cbranch_execnz .LBB47_3
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -7441,20 +7940,25 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB47_3: ; %end
+; SI-NEXT: .LBB47_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; SI-NEXT: s_branch .LBB47_2
;
; VI-LABEL: bitcast_v8bf16_to_v4f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -7531,9 +8035,7 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v1, v5
; VI-NEXT: v_mov_b32_e32 v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7544,10 +8046,18 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -7627,9 +8137,7 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7641,11 +8149,16 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-TRUE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -7727,8 +8240,6 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB47_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB47_2
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7739,11 +8250,16 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-FAKE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 16
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s3
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -7830,8 +8346,6 @@ define inreg <4 x float> @bitcast_v8bf16_to_v4f32_scalar(<8 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v7, 16, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB47_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB47_2
; GFX11-FAKE16-NEXT: .LBB47_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8180,7 +8694,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s24, s19, 24
; SI-NEXT: s_lshr_b32 s26, s19, 16
@@ -8194,8 +8708,28 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s17, 1.0
; SI-NEXT: v_add_f32_e64 v20, s16, 1.0
; SI-NEXT: v_add_f32_e64 v19, s19, 1.0
@@ -8212,22 +8746,8 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v21
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v21
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v21
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v20, s16
; SI-NEXT: v_mov_b32_e32 v21, s17
; SI-NEXT: v_mov_b32_e32 v18, s18
@@ -8244,7 +8764,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v20
; SI-NEXT: v_mov_b32_e32 v4, v21
@@ -8257,7 +8777,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s19, 24
; VI-NEXT: s_lshr_b32 s11, s19, 16
@@ -8271,8 +8791,28 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; VI-NEXT: s_lshr_b32 s23, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s17, 1.0
; VI-NEXT: v_add_f32_e64 v18, s16, 1.0
; VI-NEXT: v_add_f32_e64 v17, s19, 1.0
@@ -8289,22 +8829,8 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v18, s16
; VI-NEXT: v_mov_b32_e32 v19, s17
; VI-NEXT: v_mov_b32_e32 v16, s18
@@ -8321,7 +8847,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v15, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_mov_b32_e32 v0, v18
; VI-NEXT: v_mov_b32_e32 v4, v19
; VI-NEXT: v_mov_b32_e32 v8, v16
@@ -8332,7 +8858,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s19, 24
; GFX9-NEXT: s_lshr_b32 s11, s19, 16
@@ -8346,8 +8872,28 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s16, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s19, 1.0
@@ -8364,22 +8910,8 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v18, s16
; GFX9-NEXT: v_mov_b32_e32 v19, s17
; GFX9-NEXT: v_mov_b32_e32 v16, s18
@@ -8396,7 +8928,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v18
; GFX9-NEXT: v_mov_b32_e32 v4, v19
; GFX9-NEXT: v_mov_b32_e32 v8, v16
@@ -8408,7 +8940,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s9, s3, 24
; GFX11-NEXT: s_lshr_b32 s10, s3, 16
@@ -8422,9 +8954,28 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB49_3
+; GFX11-NEXT: .LBB49_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr18
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: .LBB49_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v19, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v17, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s2, 1.0
@@ -8443,22 +8994,8 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: s_branch .LBB49_2
-; GFX11-NEXT: .LBB49_4:
+; GFX11-NEXT: s_branch .LBB49_6
+; GFX11-NEXT: .LBB49_5:
; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
@@ -8469,7 +9006,7 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v15, s9
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB49_5: ; %end
+; GFX11-NEXT: .LBB49_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v18
; GFX11-NEXT: v_mov_b32_e32 v4, v19
; GFX11-NEXT: v_mov_b32_e32 v8, v16
@@ -9025,7 +9562,7 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -9063,8 +9600,17 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -9117,15 +9663,12 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v16i8_to_v4f32_scalar:
; VI: ; %bb.0:
@@ -9134,7 +9677,7 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -9161,8 +9704,17 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -9204,11 +9756,8 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v16i8_to_v4f32_scalar:
; GFX9: ; %bb.0:
@@ -9217,7 +9766,7 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -9244,8 +9793,17 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB51_3
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -9284,18 +9842,15 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB51_3: ; %end
+; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB51_2
;
; GFX11-LABEL: bitcast_v16i8_to_v4f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9317,9 +9872,17 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB51_3
+; GFX11-NEXT: .LBB51_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB51_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -9358,11 +9921,8 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB51_3: ; %end
+; GFX11-NEXT: .LBB51_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB51_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9468,85 +10028,106 @@ define inreg <2 x double> @bitcast_v2i64_to_v2f64_scalar(<2 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v2i64_to_v2f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v2i64_to_v2f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v2i64_to_v2f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9646,16 +10227,22 @@ define inreg <2 x i64> @bitcast_v2f64_to_v2i64_scalar(<2 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -9666,16 +10253,22 @@ define inreg <2 x i64> @bitcast_v2f64_to_v2i64_scalar(<2 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9686,16 +10279,22 @@ define inreg <2 x i64> @bitcast_v2f64_to_v2i64_scalar(<2 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9707,16 +10306,19 @@ define inreg <2 x i64> @bitcast_v2f64_to_v2i64_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9852,14 +10454,26 @@ define inreg <8 x i16> @bitcast_v2i64_to_v8i16_scalar(<2 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
@@ -9868,7 +10482,7 @@ define inreg <8 x i16> @bitcast_v2i64_to_v8i16_scalar(<2 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -9886,76 +10500,85 @@ define inreg <8 x i16> @bitcast_v2i64_to_v8i16_scalar(<2 x i64> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v2i64_to_v8i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v2i64_to_v8i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_3
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB57_3: ; %end
+; GFX9-NEXT: .LBB57_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: s_branch .LBB57_2
;
; GFX11-LABEL: bitcast_v2i64_to_v8i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_3
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB57_3: ; %end
+; GFX11-NEXT: .LBB57_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_4:
-; GFX11-NEXT: s_branch .LBB57_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10123,7 +10746,7 @@ define inreg <2 x i64> @bitcast_v8i16_to_v2i64_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -10137,8 +10760,17 @@ define inreg <2 x i64> @bitcast_v8i16_to_v2i64_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -10159,24 +10791,29 @@ define inreg <2 x i64> @bitcast_v8i16_to_v2i64_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v8i16_to_v2i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s19, 3
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -10197,31 +10834,35 @@ define inreg <2 x i64> @bitcast_v8i16_to_v2i64_scalar(<8 x i16> inreg %a, i32 in
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v8i16_to_v2i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10233,18 +10874,21 @@ define inreg <2 x i64> @bitcast_v8i16_to_v2i64_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10380,14 +11024,26 @@ define inreg <8 x half> @bitcast_v2i64_to_v8f16_scalar(<2 x i64> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB61_4
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s19, 16
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB61_3
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
@@ -10396,7 +11052,7 @@ define inreg <8 x half> @bitcast_v2i64_to_v8f16_scalar(<2 x i64> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: .LBB61_3: ; %end
+; SI-NEXT: .LBB61_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -10414,76 +11070,85 @@ define inreg <8 x half> @bitcast_v2i64_to_v8f16_scalar(<2 x i64> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB61_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB61_2
;
; VI-LABEL: bitcast_v2i64_to_v8f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB61_4
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_3
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB61_3: ; %end
+; VI-NEXT: .LBB61_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_4:
-; VI-NEXT: s_branch .LBB61_2
;
; GFX9-LABEL: bitcast_v2i64_to_v8f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_3
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB61_3: ; %end
+; GFX9-NEXT: .LBB61_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_4:
-; GFX9-NEXT: s_branch .LBB61_2
;
; GFX11-LABEL: bitcast_v2i64_to_v8f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_3
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB61_3: ; %end
+; GFX11-NEXT: .LBB61_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_4:
-; GFX11-NEXT: s_branch .LBB61_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10668,7 +11333,7 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB63_3
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -10682,8 +11347,17 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB63_4
-; SI-NEXT: .LBB63_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -10717,10 +11391,7 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB63_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB63_2
-; SI-NEXT: .LBB63_4:
+; SI-NEXT: .LBB63_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -10731,10 +11402,18 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -10757,9 +11436,7 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10770,19 +11447,25 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10794,18 +11477,21 @@ define inreg <2 x i64> @bitcast_v8f16_to_v2i64_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB63_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB63_3:
-; GFX11-NEXT: s_branch .LBB63_2
; GFX11-NEXT: .LBB63_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10962,7 +11648,7 @@ define inreg <8 x bfloat> @bitcast_v2i64_to_v8bf16_scalar(<2 x i64> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB65_4
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s19, 16
@@ -10972,8 +11658,24 @@ define inreg <8 x bfloat> @bitcast_v2i64_to_v8bf16_scalar(<2 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s10, s17, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s12, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB65_3
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s16, 3
; SI-NEXT: s_addc_u32 s5, s17, 0
; SI-NEXT: s_add_u32 s8, s18, 3
@@ -10986,7 +11688,7 @@ define inreg <8 x bfloat> @bitcast_v2i64_to_v8bf16_scalar(<2 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s10, s5, 16
; SI-NEXT: s_and_b32 s13, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s12, s4, 16
-; SI-NEXT: .LBB65_3: ; %end
+; SI-NEXT: .LBB65_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
@@ -11004,80 +11706,85 @@ define inreg <8 x bfloat> @bitcast_v2i64_to_v8bf16_scalar(<2 x i64> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB65_2
;
; VI-LABEL: bitcast_v2i64_to_v8bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB65_4
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_3
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB65_3: ; %end
+; VI-NEXT: .LBB65_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_4:
-; VI-NEXT: s_branch .LBB65_2
;
; GFX9-LABEL: bitcast_v2i64_to_v8bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_3
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB65_3: ; %end
+; GFX9-NEXT: .LBB65_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_4:
-; GFX9-NEXT: s_branch .LBB65_2
;
; GFX11-LABEL: bitcast_v2i64_to_v8bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_3
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB65_3: ; %end
+; GFX11-NEXT: .LBB65_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_4:
-; GFX11-NEXT: s_branch .LBB65_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11542,7 +12249,7 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB67_4
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
@@ -11552,8 +12259,17 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_cbranch_execnz .LBB67_3
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -11578,20 +12294,25 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB67_3: ; %end
+; SI-NEXT: .LBB67_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB67_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; SI-NEXT: s_branch .LBB67_2
;
; VI-LABEL: bitcast_v8bf16_to_v2i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -11668,9 +12389,7 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v1, v5
; VI-NEXT: v_mov_b32_e32 v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11681,10 +12400,18 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -11764,9 +12491,7 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11778,11 +12503,16 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-TRUE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -11864,8 +12594,6 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB67_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB67_2
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11876,11 +12604,16 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-FAKE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 16
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s3
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -11967,8 +12700,6 @@ define inreg <2 x i64> @bitcast_v8bf16_to_v2i64_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v7, 16, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB67_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB67_2
; GFX11-FAKE16-NEXT: .LBB67_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12321,7 +13052,7 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB69_4
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s22, s19, 24
; SI-NEXT: s_lshr_b32 s23, s19, 16
@@ -12335,8 +13066,28 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB69_3
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
@@ -12353,7 +13104,7 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 8
-; SI-NEXT: .LBB69_3: ; %end
+; SI-NEXT: .LBB69_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_mov_b32_e32 v2, s12
@@ -12371,26 +13122,12 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v14, s23
; SI-NEXT: v_mov_b32_e32 v15, s22
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: s_branch .LBB69_2
;
; VI-LABEL: bitcast_v2i64_to_v16i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB69_4
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s19, 24
; VI-NEXT: s_lshr_b32 s11, s19, 16
@@ -12404,8 +13141,28 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s23, s16, 8
; VI-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB69_3
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -12422,7 +13179,7 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s21, s17, 8
; VI-NEXT: s_lshr_b32 s22, s16, 16
; VI-NEXT: s_lshr_b32 s23, s16, 8
-; VI-NEXT: .LBB69_3: ; %end
+; VI-NEXT: .LBB69_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s23
; VI-NEXT: v_mov_b32_e32 v2, s22
@@ -12440,26 +13197,12 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v14, s11
; VI-NEXT: v_mov_b32_e32 v15, s10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_4:
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB69_2
;
; GFX9-LABEL: bitcast_v2i64_to_v16i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s19, 24
; GFX9-NEXT: s_lshr_b32 s11, s19, 16
@@ -12473,8 +13216,28 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
; GFX9-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB69_3
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -12491,7 +13254,7 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s21, s17, 8
; GFX9-NEXT: s_lshr_b32 s22, s16, 16
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
-; GFX9-NEXT: .LBB69_3: ; %end
+; GFX9-NEXT: .LBB69_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s23
; GFX9-NEXT: v_mov_b32_e32 v2, s22
@@ -12509,27 +13272,13 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v14, s11
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_4:
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB69_2
;
; GFX11-LABEL: bitcast_v2i64_to_v16i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s18, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s3, 24
; GFX11-NEXT: s_lshr_b32 s9, s3, 16
@@ -12543,9 +13292,28 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s18
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_3
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s18, -1
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -12562,7 +13330,7 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s15, s1, 8
; GFX11-NEXT: s_lshr_b32 s16, s0, 16
; GFX11-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-NEXT: .LBB69_3: ; %end
+; GFX11-NEXT: .LBB69_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17
; GFX11-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v3, s6
@@ -12573,20 +13341,6 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX11-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s10
; GFX11-NEXT: v_dual_mov_b32 v14, s9 :: v_dual_mov_b32 v15, s8
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_4:
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB69_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -13137,7 +13891,7 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -13175,8 +13929,17 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -13229,15 +13992,12 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v16i8_to_v2i64_scalar:
; VI: ; %bb.0:
@@ -13246,7 +14006,7 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -13273,8 +14033,17 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -13316,11 +14085,8 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v16i8_to_v2i64_scalar:
; GFX9: ; %bb.0:
@@ -13329,7 +14095,7 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -13356,8 +14122,17 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -13396,18 +14171,15 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-LABEL: bitcast_v16i8_to_v2i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13429,9 +14201,17 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -13470,11 +14250,8 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB71_3: ; %end
+; GFX11-NEXT: .LBB71_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB71_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -13600,28 +14377,34 @@ define inreg <8 x i16> @bitcast_v2f64_to_v8i16_scalar(<2 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB73_3
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s11, s19, 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB73_4
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: s_branch .LBB73_5
-; SI-NEXT: .LBB73_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB73_2
-; SI-NEXT: .LBB73_4:
+; SI-NEXT: s_branch .LBB73_6
+; SI-NEXT: .LBB73_5:
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
@@ -13630,7 +14413,7 @@ define inreg <8 x i16> @bitcast_v2f64_to_v8i16_scalar(<2 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -13649,16 +14432,22 @@ define inreg <8 x i16> @bitcast_v2f64_to_v8i16_scalar(<2 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
+; VI-NEXT: .LBB73_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13669,16 +14458,22 @@ define inreg <8 x i16> @bitcast_v2f64_to_v8i16_scalar(<2 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13690,16 +14485,19 @@ define inreg <8 x i16> @bitcast_v2f64_to_v8i16_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB73_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB73_3:
-; GFX11-NEXT: s_branch .LBB73_2
; GFX11-NEXT: .LBB73_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13871,7 +14669,7 @@ define inreg <2 x double> @bitcast_v8i16_to_v2f64_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_4
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -13885,8 +14683,17 @@ define inreg <2 x double> @bitcast_v8i16_to_v2f64_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB75_3
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -13907,24 +14714,29 @@ define inreg <2 x double> @bitcast_v8i16_to_v2f64_scalar(<8 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
-; SI-NEXT: .LBB75_3: ; %end
+; SI-NEXT: .LBB75_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB75_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB75_2
;
; VI-LABEL: bitcast_v8i16_to_v2f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB75_4
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB75_3
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s19, 3
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -13945,31 +14757,35 @@ define inreg <2 x double> @bitcast_v8i16_to_v2f64_scalar(<8 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB75_3: ; %end
+; VI-NEXT: .LBB75_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB75_4:
-; VI-NEXT: s_branch .LBB75_2
;
; GFX9-LABEL: bitcast_v8i16_to_v2f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB75_4
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB75_3:
-; GFX9-NEXT: s_branch .LBB75_2
-; GFX9-NEXT: .LBB75_4:
+; GFX9-NEXT: .LBB75_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13981,18 +14797,21 @@ define inreg <2 x double> @bitcast_v8i16_to_v2f64_scalar(<8 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB75_4
-; GFX11-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB75_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: s_branch .LBB75_2
; GFX11-NEXT: .LBB75_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -14122,28 +14941,34 @@ define inreg <8 x half> @bitcast_v2f64_to_v8f16_scalar(<2 x double> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB77_3
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s11, s19, 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB77_4
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: s_branch .LBB77_5
-; SI-NEXT: .LBB77_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB77_2
-; SI-NEXT: .LBB77_4:
+; SI-NEXT: s_branch .LBB77_6
+; SI-NEXT: .LBB77_5:
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
@@ -14152,7 +14977,7 @@ define inreg <8 x half> @bitcast_v2f64_to_v8f16_scalar(<2 x double> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: .LBB77_5: ; %end
+; SI-NEXT: .LBB77_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -14171,16 +14996,22 @@ define inreg <8 x half> @bitcast_v2f64_to_v8f16_scalar(<2 x double> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14191,16 +15022,22 @@ define inreg <8 x half> @bitcast_v2f64_to_v8f16_scalar(<2 x double> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14212,16 +15049,19 @@ define inreg <8 x half> @bitcast_v2f64_to_v8f16_scalar(<2 x double> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB77_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB77_3:
-; GFX11-NEXT: s_branch .LBB77_2
; GFX11-NEXT: .LBB77_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -14410,7 +15250,7 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_3
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -14424,8 +15264,17 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; SI-NEXT: s_and_b32 s7, s19, 0xffff
; SI-NEXT: s_lshl_b32 s8, s10, 16
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB79_4
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -14459,10 +15308,7 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB79_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB79_2
-; SI-NEXT: .LBB79_4:
+; SI-NEXT: .LBB79_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -14473,10 +15319,18 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB79_3
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB79_4
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -14499,9 +15353,7 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB79_3:
-; VI-NEXT: s_branch .LBB79_2
-; VI-NEXT: .LBB79_4:
+; VI-NEXT: .LBB79_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14512,19 +15364,25 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB79_4
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB79_3:
-; GFX9-NEXT: s_branch .LBB79_2
-; GFX9-NEXT: .LBB79_4:
+; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14536,18 +15394,21 @@ define inreg <2 x double> @bitcast_v8f16_to_v2f64_scalar(<8 x half> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB79_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_3:
-; GFX11-NEXT: s_branch .LBB79_2
; GFX11-NEXT: .LBB79_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -14696,7 +15557,7 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s13, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s12, s19, 16
@@ -14706,8 +15567,24 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; SI-NEXT: s_lshl_b32 s8, s17, 16
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[16:17], 1.0
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
@@ -14718,18 +15595,8 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; SI-NEXT: s_branch .LBB81_5
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: s_branch .LBB81_6
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v5, s13
; SI-NEXT: v_mov_b32_e32 v4, s12
; SI-NEXT: v_mov_b32_e32 v6, s11
@@ -14738,7 +15605,7 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: .LBB81_5: ; %end
+; SI-NEXT: .LBB81_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14761,16 +15628,22 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14781,16 +15654,22 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14802,16 +15681,19 @@ define inreg <8 x bfloat> @bitcast_v2f64_to_v8bf16_scalar(<2 x double> inreg %a,
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15280,7 +16162,7 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
; SI-NEXT: v_lshr_b64 v[0:1], v[11:12], 16
@@ -15290,8 +16172,17 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_lshr_b64 v[2:3], v[7:8], 16
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v13
; SI-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -15316,20 +16207,25 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v8bf16_to_v2f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB83_3
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_4
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -15406,9 +16302,7 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v1, v5
; VI-NEXT: v_mov_b32_e32 v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB83_3:
-; VI-NEXT: s_branch .LBB83_2
-; VI-NEXT: .LBB83_4:
+; VI-NEXT: .LBB83_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15419,10 +16313,18 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_4
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s19
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -15502,9 +16404,7 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB83_3:
-; GFX9-NEXT: s_branch .LBB83_2
-; GFX9-NEXT: .LBB83_4:
+; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15516,11 +16416,16 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-TRUE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -15602,8 +16507,6 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB83_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB83_2
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15614,11 +16517,16 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-FAKE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 16
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s3
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -15705,8 +16613,6 @@ define inreg <2 x double> @bitcast_v8bf16_to_v2f64_scalar(<8 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v7, 16, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB83_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB83_2
; GFX11-FAKE16-NEXT: .LBB83_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -16051,7 +16957,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB85_3
+; SI-NEXT: s_cbranch_scc0 .LBB85_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s27, s19, 24
; SI-NEXT: s_lshr_b32 s26, s19, 16
@@ -16065,8 +16971,28 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB85_4
-; SI-NEXT: .LBB85_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB85_3
+; SI-NEXT: .LBB85_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: .LBB85_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB85_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[20:21], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[18:19], 1.0
; SI-NEXT: v_lshr_b64 v[0:1], v[20:21], 16
@@ -16081,22 +17007,8 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v21
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v21
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v21
-; SI-NEXT: s_branch .LBB85_5
-; SI-NEXT: .LBB85_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: s_branch .LBB85_2
-; SI-NEXT: .LBB85_4:
+; SI-NEXT: s_branch .LBB85_6
+; SI-NEXT: .LBB85_5:
; SI-NEXT: v_mov_b32_e32 v19, s19
; SI-NEXT: v_mov_b32_e32 v21, s17
; SI-NEXT: v_mov_b32_e32 v20, s16
@@ -16113,7 +17025,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v9, s8
; SI-NEXT: v_mov_b32_e32 v16, s6
; SI-NEXT: v_mov_b32_e32 v11, s4
-; SI-NEXT: .LBB85_5: ; %end
+; SI-NEXT: .LBB85_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v20
; SI-NEXT: v_mov_b32_e32 v4, v21
@@ -16126,7 +17038,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB85_3
+; VI-NEXT: s_cbranch_scc0 .LBB85_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s15, s19, 24
; VI-NEXT: s_lshr_b32 s14, s19, 16
@@ -16140,24 +17052,10 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; VI-NEXT: s_lshr_b32 s22, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB85_4
-; VI-NEXT: .LBB85_2: ; %cmp.true
-; VI-NEXT: v_add_f64 v[16:17], s[18:19], 1.0
-; VI-NEXT: v_add_f64 v[18:19], s[16:17], 1.0
-; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
-; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; VI-NEXT: v_lshrrev_b32_e32 v15, 24, v17
-; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v17
-; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; VI-NEXT: v_lshrrev_b32_e32 v7, 24, v19
-; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; VI-NEXT: s_branch .LBB85_5
-; VI-NEXT: .LBB85_3:
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB85_3
+; VI-NEXT: .LBB85_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
; VI-NEXT: ; implicit-def: $sgpr22
; VI-NEXT: ; implicit-def: $sgpr23
; VI-NEXT: ; implicit-def: $sgpr4
@@ -16170,8 +17068,28 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; VI-NEXT: ; implicit-def: $sgpr13
; VI-NEXT: ; implicit-def: $sgpr14
; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: s_branch .LBB85_2
-; VI-NEXT: .LBB85_4:
+; VI-NEXT: .LBB85_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB85_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
+; VI-NEXT: v_add_f64 v[16:17], s[18:19], 1.0
+; VI-NEXT: v_add_f64 v[18:19], s[16:17], 1.0
+; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; VI-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; VI-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; VI-NEXT: s_branch .LBB85_6
+; VI-NEXT: .LBB85_5:
; VI-NEXT: v_mov_b32_e32 v18, s16
; VI-NEXT: v_mov_b32_e32 v16, s18
; VI-NEXT: v_mov_b32_e32 v17, s19
@@ -16188,7 +17106,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v7, s12
; VI-NEXT: v_mov_b32_e32 v6, s11
; VI-NEXT: v_mov_b32_e32 v5, s10
-; VI-NEXT: .LBB85_5: ; %end
+; VI-NEXT: .LBB85_6: ; %end
; VI-NEXT: v_mov_b32_e32 v0, v18
; VI-NEXT: v_mov_b32_e32 v4, v19
; VI-NEXT: v_mov_b32_e32 v8, v16
@@ -16199,7 +17117,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB85_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s15, s19, 24
; GFX9-NEXT: s_lshr_b32 s14, s19, 16
@@ -16213,8 +17131,28 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s22, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB85_4
-; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB85_3
+; GFX9-NEXT: .LBB85_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: .LBB85_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[16:17], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[16:17], 1.0
; GFX9-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
@@ -16229,22 +17167,8 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX9-NEXT: s_branch .LBB85_5
-; GFX9-NEXT: .LBB85_3:
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: s_branch .LBB85_2
-; GFX9-NEXT: .LBB85_4:
+; GFX9-NEXT: s_branch .LBB85_6
+; GFX9-NEXT: .LBB85_5:
; GFX9-NEXT: v_mov_b32_e32 v18, s16
; GFX9-NEXT: v_mov_b32_e32 v16, s18
; GFX9-NEXT: v_mov_b32_e32 v17, s19
@@ -16261,7 +17185,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v7, s12
; GFX9-NEXT: v_mov_b32_e32 v6, s11
; GFX9-NEXT: v_mov_b32_e32 v5, s10
-; GFX9-NEXT: .LBB85_5: ; %end
+; GFX9-NEXT: .LBB85_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v18
; GFX9-NEXT: v_mov_b32_e32 v4, v19
; GFX9-NEXT: v_mov_b32_e32 v8, v16
@@ -16273,7 +17197,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s3, 24
; GFX11-NEXT: s_lshr_b32 s13, s3, 16
@@ -16287,9 +17211,28 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
-; GFX11-NEXT: .LBB85_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB85_3
+; GFX11-NEXT: .LBB85_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr18
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB85_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[16:17], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[0:1], 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -16305,22 +17248,8 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB85_5
-; GFX11-NEXT: .LBB85_3:
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB85_2
-; GFX11-NEXT: .LBB85_4:
+; GFX11-NEXT: s_branch .LBB85_6
+; GFX11-NEXT: .LBB85_5:
; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v17, s3
; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v19, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
@@ -16331,7 +17260,7 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v13, s12
; GFX11-NEXT: v_mov_b32_e32 v7, s11
; GFX11-NEXT: v_mov_b32_e32 v5, s9
-; GFX11-NEXT: .LBB85_5: ; %end
+; GFX11-NEXT: .LBB85_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v18
; GFX11-NEXT: v_mov_b32_e32 v4, v19
; GFX11-NEXT: v_mov_b32_e32 v8, v16
@@ -16887,7 +17816,7 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB87_4
+; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -16925,8 +17854,17 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_cbranch_execnz .LBB87_3
-; SI-NEXT: .LBB87_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB87_3
+; SI-NEXT: .LBB87_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
+; SI-NEXT: .LBB87_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB87_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -16979,15 +17917,12 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: .LBB87_3: ; %end
+; SI-NEXT: .LBB87_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB87_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7
-; SI-NEXT: s_branch .LBB87_2
;
; VI-LABEL: bitcast_v16i8_to_v2f64_scalar:
; VI: ; %bb.0:
@@ -16996,7 +17931,7 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB87_4
+; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -17023,8 +17958,17 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB87_3
-; VI-NEXT: .LBB87_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB87_3
+; VI-NEXT: .LBB87_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB87_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB87_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -17066,11 +18010,8 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB87_3: ; %end
+; VI-NEXT: .LBB87_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB87_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB87_2
;
; GFX9-LABEL: bitcast_v16i8_to_v2f64_scalar:
; GFX9: ; %bb.0:
@@ -17079,7 +18020,7 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -17106,8 +18047,17 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB87_3
-; GFX9-NEXT: .LBB87_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB87_3
+; GFX9-NEXT: .LBB87_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB87_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -17146,18 +18096,15 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB87_3: ; %end
+; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB87_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB87_2
;
; GFX11-LABEL: bitcast_v16i8_to_v2f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -17179,9 +18126,17 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB87_3
-; GFX11-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB87_3
+; GFX11-NEXT: .LBB87_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB87_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -17220,11 +18175,8 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB87_3: ; %end
+; GFX11-NEXT: .LBB87_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB87_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB87_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -17408,7 +18360,7 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s21, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB89_4
+; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
@@ -17424,10 +18376,22 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; SI-NEXT: s_or_b32 s4, s4, s26
; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[26:27], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_mov_b32 s7, s25
; SI-NEXT: s_mov_b32 s5, s27
-; SI-NEXT: s_cbranch_execnz .LBB89_3
-; SI-NEXT: .LBB89_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB89_3
+; SI-NEXT: .LBB89_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB89_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB89_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_and_b32 s4, s18, 0xffff
; SI-NEXT: s_lshl_b32 s5, s22, 16
@@ -17452,7 +18416,7 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
; SI-NEXT: s_lshr_b32 s14, s7, 16
; SI-NEXT: s_lshr_b32 s15, s5, 16
-; SI-NEXT: .LBB89_3: ; %end
+; SI-NEXT: .LBB89_5: ; %end
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_or_b32 s6, s6, s8
@@ -17470,21 +18434,23 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB89_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB89_2
;
; VI-LABEL: bitcast_v8i16_to_v8f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB89_4
+; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB89_3
-; VI-NEXT: .LBB89_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB89_3
+; VI-NEXT: .LBB89_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB89_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB89_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -17505,31 +18471,35 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB89_3: ; %end
+; VI-NEXT: .LBB89_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB89_4:
-; VI-NEXT: s_branch .LBB89_2
;
; GFX9-LABEL: bitcast_v8i16_to_v8f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB89_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB89_4
-; GFX9-NEXT: .LBB89_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB89_3
+; GFX9-NEXT: .LBB89_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB89_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB89_3:
-; GFX9-NEXT: s_branch .LBB89_2
-; GFX9-NEXT: .LBB89_4:
+; GFX9-NEXT: .LBB89_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17541,18 +18511,21 @@ define inreg <8 x half> @bitcast_v8i16_to_v8f16_scalar(<8 x i16> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB89_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB89_4
-; GFX11-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB89_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB89_3:
-; GFX11-NEXT: s_branch .LBB89_2
; GFX11-NEXT: .LBB89_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -17723,10 +18696,18 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB91_3
+; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB91_4
-; SI-NEXT: .LBB91_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB91_3
+; SI-NEXT: .LBB91_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB91_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB91_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
@@ -17761,10 +18742,8 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
; SI-NEXT: v_or_b32_e32 v2, v11, v2
-; SI-NEXT: s_branch .LBB91_5
-; SI-NEXT: .LBB91_3:
-; SI-NEXT: s_branch .LBB91_2
-; SI-NEXT: .LBB91_4:
+; SI-NEXT: s_branch .LBB91_6
+; SI-NEXT: .LBB91_5:
; SI-NEXT: v_mov_b32_e32 v8, s7
; SI-NEXT: v_mov_b32_e32 v10, s6
; SI-NEXT: v_mov_b32_e32 v1, s17
@@ -17773,7 +18752,7 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v9, s16
; SI-NEXT: v_mov_b32_e32 v6, s9
; SI-NEXT: v_mov_b32_e32 v4, s8
-; SI-NEXT: .LBB91_5: ; %end
+; SI-NEXT: .LBB91_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -17792,10 +18771,18 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB91_3
+; VI-NEXT: s_cbranch_scc0 .LBB91_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB91_4
-; VI-NEXT: .LBB91_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB91_3
+; VI-NEXT: .LBB91_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB91_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB91_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -17818,9 +18805,7 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: v_or_b32_e32 v1, v1, v6
; VI-NEXT: v_or_b32_e32 v0, v4, v5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB91_3:
-; VI-NEXT: s_branch .LBB91_2
-; VI-NEXT: .LBB91_4:
+; VI-NEXT: .LBB91_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17831,19 +18816,25 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB91_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB91_4
-; GFX9-NEXT: .LBB91_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB91_3
+; GFX9-NEXT: .LBB91_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB91_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB91_3:
-; GFX9-NEXT: s_branch .LBB91_2
-; GFX9-NEXT: .LBB91_4:
+; GFX9-NEXT: .LBB91_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17855,18 +18846,21 @@ define inreg <8 x i16> @bitcast_v8f16_to_v8i16_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB91_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB91_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB91_3:
-; GFX11-NEXT: s_branch .LBB91_2
; GFX11-NEXT: .LBB91_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -18049,7 +19043,7 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s15, s17, 16
; SI-NEXT: s_lshr_b32 s14, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB93_4
+; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s10, s16, 16
; SI-NEXT: s_lshl_b32 s13, s14, 16
@@ -18059,8 +19053,24 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s11, s21, 16
; SI-NEXT: s_lshl_b32 s6, s19, 16
; SI-NEXT: s_lshl_b32 s9, s22, 16
-; SI-NEXT: s_cbranch_execnz .LBB93_3
-; SI-NEXT: .LBB93_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB93_3
+; SI-NEXT: .LBB93_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB93_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB93_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_and_b32 s4, s19, 0xffff
; SI-NEXT: s_lshl_b32 s5, s22, 16
@@ -18089,7 +19099,7 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s4, 16
-; SI-NEXT: .LBB93_3: ; %end
+; SI-NEXT: .LBB93_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s13
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s10
@@ -18107,25 +19117,23 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB93_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB93_2
;
; VI-LABEL: bitcast_v8i16_to_v8bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB93_4
+; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB93_3
-; VI-NEXT: .LBB93_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB93_3
+; VI-NEXT: .LBB93_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB93_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB93_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -18146,31 +19154,35 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB93_3: ; %end
+; VI-NEXT: .LBB93_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB93_4:
-; VI-NEXT: s_branch .LBB93_2
;
; GFX9-LABEL: bitcast_v8i16_to_v8bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB93_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB93_4
-; GFX9-NEXT: .LBB93_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB93_3
+; GFX9-NEXT: .LBB93_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB93_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB93_3:
-; GFX9-NEXT: s_branch .LBB93_2
-; GFX9-NEXT: .LBB93_4:
+; GFX9-NEXT: .LBB93_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -18182,18 +19194,21 @@ define inreg <8 x bfloat> @bitcast_v8i16_to_v8bf16_scalar(<8 x i16> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB93_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB93_4
-; GFX11-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB93_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB93_3:
-; GFX11-NEXT: s_branch .LBB93_2
; GFX11-NEXT: .LBB93_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -18671,7 +19686,7 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB95_4
+; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v15
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
@@ -18679,10 +19694,26 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v14
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v8
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; SI-NEXT: s_cbranch_execnz .LBB95_3
-; SI-NEXT: .LBB95_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB95_3
+; SI-NEXT: .LBB95_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: .LBB95_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB95_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v1
@@ -18711,7 +19742,7 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
; SI-NEXT: v_lshr_b64 v[12:13], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[13:14], v[2:3], 16
-; SI-NEXT: .LBB95_3: ; %end
+; SI-NEXT: .LBB95_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v12
; SI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -18725,25 +19756,23 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v9
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB95_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: s_branch .LBB95_2
;
; VI-LABEL: bitcast_v8bf16_to_v8i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB95_3
+; VI-NEXT: s_cbranch_scc0 .LBB95_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB95_4
-; VI-NEXT: .LBB95_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB95_3
+; VI-NEXT: .LBB95_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB95_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB95_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -18820,9 +19849,7 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: v_mov_b32_e32 v3, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB95_3:
-; VI-NEXT: s_branch .LBB95_2
-; VI-NEXT: .LBB95_4:
+; VI-NEXT: .LBB95_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -18833,10 +19860,18 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB95_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB95_4
-; GFX9-NEXT: .LBB95_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB95_3
+; GFX9-NEXT: .LBB95_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB95_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -18912,9 +19947,7 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX9-NEXT: v_and_or_b32 v0, v4, v8, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB95_3:
-; GFX9-NEXT: s_branch .LBB95_2
-; GFX9-NEXT: .LBB95_4:
+; GFX9-NEXT: .LBB95_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -18926,11 +19959,16 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-TRUE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -19001,8 +20039,6 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB95_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB95_2
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -19013,11 +20049,16 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-FAKE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -19100,8 +20141,6 @@ define inreg <8 x i16> @bitcast_v8bf16_to_v8i16_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v6, v8
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v0, v9
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB95_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB95_2
; GFX11-FAKE16-NEXT: .LBB95_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -19512,7 +20551,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s28, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB97_4
+; SI-NEXT: s_cbranch_scc0 .LBB97_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s28, 16
@@ -19536,8 +20575,28 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s15, s7, 8
; SI-NEXT: s_bfe_u32 s11, s26, 0x80008
; SI-NEXT: s_bfe_u32 s13, s27, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB97_3
-; SI-NEXT: .LBB97_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[24:25], 0
+; SI-NEXT: s_branch .LBB97_3
+; SI-NEXT: .LBB97_2:
+; SI-NEXT: s_mov_b64 s[24:25], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: .LBB97_3: ; %Flow
+; SI-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; SI-NEXT: s_cselect_b32 s21, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s21, 1
+; SI-NEXT: s_cbranch_scc1 .LBB97_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_and_b32 s4, s18, 0xffff
; SI-NEXT: s_lshl_b32 s5, s29, 16
@@ -19570,7 +20629,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s13, s7, 24
; SI-NEXT: s_lshr_b32 s27, s7, 16
; SI-NEXT: s_lshr_b32 s15, s7, 8
-; SI-NEXT: .LBB97_3: ; %end
+; SI-NEXT: .LBB97_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s12
; SI-NEXT: v_mov_b32_e32 v2, s10
@@ -19588,26 +20647,12 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v14, s27
; SI-NEXT: v_mov_b32_e32 v15, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB97_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: s_branch .LBB97_2
;
; VI-LABEL: bitcast_v8i16_to_v16i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB97_4
+; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s19, 24
; VI-NEXT: s_lshr_b32 s11, s19, 16
@@ -19621,8 +20666,28 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s23, s16, 8
; VI-NEXT: s_lshr_b64 s[4:5], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB97_3
-; VI-NEXT: .LBB97_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB97_3
+; VI-NEXT: .LBB97_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB97_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB97_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -19655,7 +20720,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s21, s17, 8
; VI-NEXT: s_lshr_b32 s22, s16, 16
; VI-NEXT: s_lshr_b32 s23, s16, 8
-; VI-NEXT: .LBB97_3: ; %end
+; VI-NEXT: .LBB97_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s23
; VI-NEXT: v_mov_b32_e32 v2, s22
@@ -19673,26 +20738,12 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v14, s11
; VI-NEXT: v_mov_b32_e32 v15, s10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB97_4:
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB97_2
;
; GFX9-LABEL: bitcast_v8i16_to_v16i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s19, 24
; GFX9-NEXT: s_lshr_b32 s11, s19, 16
@@ -19706,8 +20757,28 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB97_4
-; GFX9-NEXT: .LBB97_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB97_3
+; GFX9-NEXT: .LBB97_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB97_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v19, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v18, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s19, 3 op_sel_hi:[1,0]
@@ -19724,22 +20795,8 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX9-NEXT: s_branch .LBB97_5
-; GFX9-NEXT: .LBB97_3:
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB97_2
-; GFX9-NEXT: .LBB97_4:
+; GFX9-NEXT: s_branch .LBB97_6
+; GFX9-NEXT: .LBB97_5:
; GFX9-NEXT: v_mov_b32_e32 v18, s16
; GFX9-NEXT: v_mov_b32_e32 v19, s17
; GFX9-NEXT: v_mov_b32_e32 v16, s18
@@ -19756,7 +20813,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB97_5: ; %end
+; GFX9-NEXT: .LBB97_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v18
; GFX9-NEXT: v_mov_b32_e32 v4, v19
; GFX9-NEXT: v_mov_b32_e32 v8, v16
@@ -19768,7 +20825,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s9, s3, 24
; GFX11-NEXT: s_lshr_b32 s10, s3, 16
@@ -19782,9 +20839,28 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB97_4
-; GFX11-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB97_3
+; GFX11-NEXT: .LBB97_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr18
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: .LBB97_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v19, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v17, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v16, s2, 3 op_sel_hi:[1,0]
@@ -19803,22 +20879,8 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB97_5
-; GFX11-NEXT: .LBB97_3:
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: s_branch .LBB97_2
-; GFX11-NEXT: .LBB97_4:
+; GFX11-NEXT: s_branch .LBB97_6
+; GFX11-NEXT: .LBB97_5:
; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
@@ -19829,7 +20891,7 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-NEXT: v_mov_b32_e32 v15, s9
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB97_5: ; %end
+; GFX11-NEXT: .LBB97_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v18
; GFX11-NEXT: v_mov_b32_e32 v4, v19
; GFX11-NEXT: v_mov_b32_e32 v8, v16
@@ -20406,7 +21468,7 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB99_4
+; SI-NEXT: s_cbranch_scc0 .LBB99_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -20448,10 +21510,24 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_lshr_b32 s11, s7, 16
; SI-NEXT: s_lshr_b32 s9, s9, 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_mov_b32 s7, s41
; SI-NEXT: s_mov_b32 s5, s43
-; SI-NEXT: s_cbranch_execnz .LBB99_3
-; SI-NEXT: .LBB99_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB99_3
+; SI-NEXT: .LBB99_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB99_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB99_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xff
; SI-NEXT: s_lshl_b32 s5, s25, 8
@@ -20508,7 +21584,7 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
; SI-NEXT: s_lshr_b32 s11, s7, 16
; SI-NEXT: s_lshr_b32 s9, s5, 16
-; SI-NEXT: .LBB99_3: ; %end
+; SI-NEXT: .LBB99_5: ; %end
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_or_b32 s6, s6, s8
@@ -20526,14 +21602,6 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB99_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB99_2
;
; VI-LABEL: bitcast_v16i8_to_v8i16_scalar:
; VI: ; %bb.0:
@@ -20542,7 +21610,7 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB99_4
+; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -20569,8 +21637,17 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB99_3
-; VI-NEXT: .LBB99_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB99_3
+; VI-NEXT: .LBB99_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB99_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB99_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -20612,11 +21689,8 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB99_3: ; %end
+; VI-NEXT: .LBB99_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB99_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB99_2
;
; GFX9-LABEL: bitcast_v16i8_to_v8i16_scalar:
; GFX9: ; %bb.0:
@@ -20625,7 +21699,7 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -20652,8 +21726,17 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB99_3
-; GFX9-NEXT: .LBB99_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB99_3
+; GFX9-NEXT: .LBB99_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB99_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -20692,18 +21775,15 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB99_3: ; %end
+; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB99_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB99_2
;
; GFX11-LABEL: bitcast_v16i8_to_v8i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -20725,9 +21805,17 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB99_3
+; GFX11-NEXT: .LBB99_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB99_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -20766,11 +21854,8 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB99_3: ; %end
+; GFX11-NEXT: .LBB99_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB99_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB99_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -20965,7 +22050,7 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s7, s17, 16
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB101_3
+; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s10, s16, 16
; SI-NEXT: s_lshl_b32 s11, s6, 16
@@ -20975,8 +22060,24 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s15, s8, 16
; SI-NEXT: s_lshl_b32 s20, s19, 16
; SI-NEXT: s_lshl_b32 s21, s9, 16
-; SI-NEXT: s_cbranch_execnz .LBB101_4
-; SI-NEXT: .LBB101_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s19
; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
@@ -21009,18 +22110,8 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-NEXT: s_branch .LBB101_5
-; SI-NEXT: .LBB101_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: s_branch .LBB101_2
-; SI-NEXT: .LBB101_4:
+; SI-NEXT: s_branch .LBB101_6
+; SI-NEXT: .LBB101_5:
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v6, s15
@@ -21029,7 +22120,7 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v2, s12
; SI-NEXT: v_mov_b32_e32 v1, s11
; SI-NEXT: v_mov_b32_e32 v0, s10
-; SI-NEXT: .LBB101_5: ; %end
+; SI-NEXT: .LBB101_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -21052,10 +22143,18 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB101_3
+; VI-NEXT: s_cbranch_scc0 .LBB101_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB101_4
-; VI-NEXT: .LBB101_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB101_3
+; VI-NEXT: .LBB101_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB101_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB101_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -21078,9 +22177,7 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v6
; VI-NEXT: v_or_b32_e32 v0, v4, v5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB101_3:
-; VI-NEXT: s_branch .LBB101_2
-; VI-NEXT: .LBB101_4:
+; VI-NEXT: .LBB101_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -21091,19 +22188,25 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB101_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB101_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB101_4
-; GFX9-NEXT: .LBB101_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB101_3
+; GFX9-NEXT: .LBB101_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB101_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB101_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB101_3:
-; GFX9-NEXT: s_branch .LBB101_2
-; GFX9-NEXT: .LBB101_4:
+; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -21115,18 +22218,21 @@ define inreg <8 x bfloat> @bitcast_v8f16_to_v8bf16_scalar(<8 x half> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB101_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB101_3:
-; GFX11-NEXT: s_branch .LBB101_2
; GFX11-NEXT: .LBB101_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -21618,7 +22724,7 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s11
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
-; SI-NEXT: s_cbranch_scc0 .LBB103_4
+; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
@@ -21629,11 +22735,25 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v22
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v23
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v22
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshr_b64 v[18:19], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[9:10], v[16:17], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[11:12], 16
-; SI-NEXT: s_cbranch_execnz .LBB103_3
-; SI-NEXT: .LBB103_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB103_3
+; SI-NEXT: .LBB103_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: .LBB103_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB103_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v2
@@ -21662,7 +22782,7 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[18:19], v[2:3], 16
-; SI-NEXT: .LBB103_3: ; %end
+; SI-NEXT: .LBB103_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v14
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -21676,23 +22796,23 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v8
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB103_4:
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: s_branch .LBB103_2
;
; VI-LABEL: bitcast_v8bf16_to_v8f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB103_3
+; VI-NEXT: s_cbranch_scc0 .LBB103_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB103_4
-; VI-NEXT: .LBB103_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB103_3
+; VI-NEXT: .LBB103_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB103_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB103_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v7, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v7
@@ -21769,9 +22889,7 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: v_mov_b32_e32 v3, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB103_3:
-; VI-NEXT: s_branch .LBB103_2
-; VI-NEXT: .LBB103_4:
+; VI-NEXT: .LBB103_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -21782,10 +22900,18 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB103_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB103_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB103_4
-; GFX9-NEXT: .LBB103_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB103_3
+; GFX9-NEXT: .LBB103_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB103_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -21865,9 +22991,7 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX9-NEXT: v_and_b32_sdwa v0, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB103_3:
-; GFX9-NEXT: s_branch .LBB103_2
-; GFX9-NEXT: .LBB103_4:
+; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -21879,11 +23003,16 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-TRUE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -21965,8 +23094,6 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB103_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB103_2
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -21977,11 +23104,16 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-FAKE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -22072,8 +23204,6 @@ define inreg <8 x half> @bitcast_v8bf16_to_v8f16_scalar(<8 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v6, 16, v7
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB103_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB103_2
; GFX11-FAKE16-NEXT: .LBB103_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22488,7 +23618,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB105_3
+; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -22512,8 +23642,28 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s9, s13, 8
; SI-NEXT: s_bfe_u32 s11, s26, 0x80008
; SI-NEXT: s_bfe_u32 s15, s28, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB105_4
-; SI-NEXT: .LBB105_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[24:25], 0
+; SI-NEXT: s_branch .LBB105_3
+; SI-NEXT: .LBB105_2:
+; SI-NEXT: s_mov_b64 s[24:25], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; SI-NEXT: s_cselect_b32 s21, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s21, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s29
; SI-NEXT: v_cvt_f32_f16_e32 v1, s18
; SI-NEXT: v_cvt_f32_f16_e32 v2, s28
@@ -22556,22 +23706,8 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_lshrrev_b32_e32 v13, 8, v18
; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
; SI-NEXT: v_bfe_u32 v15, v14, 8, 8
-; SI-NEXT: s_branch .LBB105_5
-; SI-NEXT: .LBB105_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: s_branch .LBB105_2
-; SI-NEXT: .LBB105_4:
+; SI-NEXT: s_branch .LBB105_6
+; SI-NEXT: .LBB105_5:
; SI-NEXT: v_mov_b32_e32 v14, s28
; SI-NEXT: v_mov_b32_e32 v6, s26
; SI-NEXT: v_mov_b32_e32 v15, s15
@@ -22588,7 +23724,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v11, s14
; SI-NEXT: v_mov_b32_e32 v12, s20
; SI-NEXT: v_mov_b32_e32 v9, s22
-; SI-NEXT: .LBB105_5: ; %end
+; SI-NEXT: .LBB105_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v19
; SI-NEXT: v_mov_b32_e32 v4, v20
@@ -22601,7 +23737,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB105_3
+; VI-NEXT: s_cbranch_scc0 .LBB105_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s19, 24
; VI-NEXT: s_lshr_b32 s20, s19, 16
@@ -22615,8 +23751,28 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: s_lshr_b32 s12, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB105_4
-; VI-NEXT: .LBB105_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB105_3
+; VI-NEXT: .LBB105_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB105_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB105_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_add_f16_e32 v6, s4, v1
@@ -22646,22 +23802,8 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; VI-NEXT: v_bfe_u32 v15, v14, 8, 8
; VI-NEXT: v_bfe_u32 v7, v6, 8, 8
-; VI-NEXT: s_branch .LBB105_5
-; VI-NEXT: .LBB105_3:
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB105_2
-; VI-NEXT: .LBB105_4:
+; VI-NEXT: s_branch .LBB105_6
+; VI-NEXT: .LBB105_5:
; VI-NEXT: v_mov_b32_e32 v2, s23
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v10, s21
@@ -22678,7 +23820,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; VI-NEXT: v_mov_b32_e32 v5, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB105_5: ; %end
+; VI-NEXT: .LBB105_6: ; %end
; VI-NEXT: v_mov_b32_e32 v4, v17
; VI-NEXT: v_mov_b32_e32 v12, v16
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -22687,7 +23829,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB105_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s19, 24
; GFX9-NEXT: s_lshr_b32 s11, s19, 16
@@ -22701,8 +23843,28 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX9-NEXT: s_lshr_b32 s23, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB105_4
-; GFX9-NEXT: .LBB105_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB105_3
+; GFX9-NEXT: .LBB105_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB105_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v19, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s16, v0 op_sel_hi:[1,0]
@@ -22720,22 +23882,8 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX9-NEXT: s_branch .LBB105_5
-; GFX9-NEXT: .LBB105_3:
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB105_2
-; GFX9-NEXT: .LBB105_4:
+; GFX9-NEXT: s_branch .LBB105_6
+; GFX9-NEXT: .LBB105_5:
; GFX9-NEXT: v_mov_b32_e32 v18, s16
; GFX9-NEXT: v_mov_b32_e32 v19, s17
; GFX9-NEXT: v_mov_b32_e32 v16, s18
@@ -22752,7 +23900,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX9-NEXT: v_mov_b32_e32 v15, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB105_5: ; %end
+; GFX9-NEXT: .LBB105_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v18
; GFX9-NEXT: v_mov_b32_e32 v4, v19
; GFX9-NEXT: v_mov_b32_e32 v8, v16
@@ -22764,7 +23912,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s9, s3, 24
; GFX11-NEXT: s_lshr_b32 s10, s3, 16
@@ -22778,9 +23926,28 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB105_3
+; GFX11-NEXT: .LBB105_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr18
+; GFX11-NEXT: ; implicit-def: $sgpr16
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr17
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: .LBB105_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s2 op_sel_hi:[0,1]
@@ -22799,22 +23966,8 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX11-NEXT: s_branch .LBB105_5
-; GFX11-NEXT: .LBB105_3:
-; GFX11-NEXT: ; implicit-def: $sgpr18
-; GFX11-NEXT: ; implicit-def: $sgpr16
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr17
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: s_branch .LBB105_2
-; GFX11-NEXT: .LBB105_4:
+; GFX11-NEXT: s_branch .LBB105_6
+; GFX11-NEXT: .LBB105_5:
; GFX11-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
; GFX11-NEXT: v_dual_mov_b32 v16, s2 :: v_dual_mov_b32 v17, s3
; GFX11-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
@@ -22825,7 +23978,7 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-NEXT: v_mov_b32_e32 v15, s9
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB105_5: ; %end
+; GFX11-NEXT: .LBB105_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v18
; GFX11-NEXT: v_mov_b32_e32 v4, v19
; GFX11-NEXT: v_mov_b32_e32 v8, v16
@@ -23402,7 +24555,7 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB107_4
+; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -23444,10 +24597,24 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_lshr_b32 s11, s7, 16
; SI-NEXT: s_lshr_b32 s9, s9, 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_mov_b32 s7, s41
; SI-NEXT: s_mov_b32 s5, s43
-; SI-NEXT: s_cbranch_execnz .LBB107_3
-; SI-NEXT: .LBB107_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB107_3
+; SI-NEXT: .LBB107_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB107_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB107_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xff
; SI-NEXT: s_lshl_b32 s5, s25, 8
@@ -23504,7 +24671,7 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
; SI-NEXT: s_lshr_b32 s11, s7, 16
; SI-NEXT: s_lshr_b32 s9, s5, 16
-; SI-NEXT: .LBB107_3: ; %end
+; SI-NEXT: .LBB107_5: ; %end
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_or_b32 s6, s6, s8
@@ -23522,14 +24689,6 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB107_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB107_2
;
; VI-LABEL: bitcast_v16i8_to_v8f16_scalar:
; VI: ; %bb.0:
@@ -23538,7 +24697,7 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB107_4
+; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -23565,8 +24724,17 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB107_3
-; VI-NEXT: .LBB107_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB107_3
+; VI-NEXT: .LBB107_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB107_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB107_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -23608,11 +24776,8 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB107_3: ; %end
+; VI-NEXT: .LBB107_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB107_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB107_2
;
; GFX9-LABEL: bitcast_v16i8_to_v8f16_scalar:
; GFX9: ; %bb.0:
@@ -23621,7 +24786,7 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -23648,8 +24813,17 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB107_3
-; GFX9-NEXT: .LBB107_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB107_3
+; GFX9-NEXT: .LBB107_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB107_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -23688,18 +24862,15 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB107_3: ; %end
+; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB107_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB107_2
;
; GFX11-LABEL: bitcast_v16i8_to_v8f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -23721,9 +24892,17 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB107_3
+; GFX11-NEXT: .LBB107_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB107_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -23762,11 +24941,8 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB107_3: ; %end
+; GFX11-NEXT: .LBB107_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB107_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB107_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -24456,101 +25632,107 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s10
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s8
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB109_4
+; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v30
-; SI-NEXT: v_lshr_b64 v[19:20], v[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v27
-; SI-NEXT: v_lshr_b64 v[22:23], v[25:26], 16
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v29
-; SI-NEXT: v_lshr_b64 v[20:21], v[5:6], 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[27:28], 16
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v31
+; SI-NEXT: v_lshr_b64 v[0:1], v[25:26], 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v29
; SI-NEXT: v_lshr_b64 v[23:24], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[0:1], v[19:20], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
; SI-NEXT: v_lshr_b64 v[10:11], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v27
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v29
-; SI-NEXT: v_lshrrev_b32_e32 v9, 8, v20
-; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v23
-; SI-NEXT: v_lshr_b64 v[1:2], v[19:20], 8
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
+; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 8
; SI-NEXT: v_lshr_b64 v[17:18], v[22:23], 24
; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 8
-; SI-NEXT: s_cbranch_execnz .LBB109_3
-; SI-NEXT: .LBB109_2: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v25
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v29
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v31
+; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v23
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB109_3
+; SI-NEXT: .LBB109_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: .LBB109_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB109_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[22:23], v[0:1], 16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v29
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v25
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[19:20], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v12
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v10
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_lshr_b64 v[20:21], v[5:6], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[0:1], v[19:20], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v10
; SI-NEXT: v_lshr_b64 v[10:11], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 24
-; SI-NEXT: v_lshr_b64 v[1:2], v[19:20], 8
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
+; SI-NEXT: v_lshr_b64 v[20:21], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 8
; SI-NEXT: v_lshr_b64 v[17:18], v[22:23], 24
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[22:23], 8
-; SI-NEXT: v_lshrrev_b32_e32 v9, 8, v20
+; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v1
; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v23
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; SI-NEXT: .LBB109_3: ; %end
-; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: v_mov_b32_e32 v0, v19
-; SI-NEXT: v_mov_b32_e32 v4, v20
-; SI-NEXT: v_mov_b32_e32 v5, v9
-; SI-NEXT: v_mov_b32_e32 v8, v22
+; SI-NEXT: .LBB109_5: ; %end
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v1, v8
+; SI-NEXT: v_mov_b32_e32 v2, v20
; SI-NEXT: v_mov_b32_e32 v9, v11
+; SI-NEXT: v_mov_b32_e32 v5, v19
+; SI-NEXT: v_mov_b32_e32 v8, v22
; SI-NEXT: v_mov_b32_e32 v11, v17
; SI-NEXT: v_mov_b32_e32 v12, v23
; SI-NEXT: v_mov_b32_e32 v13, v16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB109_4:
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: s_branch .LBB109_2
;
; VI-LABEL: bitcast_v8bf16_to_v16i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB109_3
+; VI-NEXT: s_cbranch_scc0 .LBB109_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s21, s19, 24
; VI-NEXT: s_lshr_b32 s20, s19, 16
@@ -24564,8 +25746,28 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: s_lshr_b32 s13, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB109_4
-; VI-NEXT: .LBB109_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB109_3
+; VI-NEXT: .LBB109_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr22
+; VI-NEXT: ; implicit-def: $sgpr23
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr20
+; VI-NEXT: ; implicit-def: $sgpr21
+; VI-NEXT: .LBB109_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB109_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v6, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v6
@@ -24653,22 +25855,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v11, 8, v0
-; VI-NEXT: s_branch .LBB109_5
-; VI-NEXT: .LBB109_3:
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr22
-; VI-NEXT: ; implicit-def: $sgpr23
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr20
-; VI-NEXT: ; implicit-def: $sgpr21
-; VI-NEXT: s_branch .LBB109_2
-; VI-NEXT: .LBB109_4:
+; VI-NEXT: s_branch .LBB109_6
+; VI-NEXT: .LBB109_5:
; VI-NEXT: v_mov_b32_e32 v8, s18
; VI-NEXT: v_mov_b32_e32 v12, s19
; VI-NEXT: v_mov_b32_e32 v0, s16
@@ -24685,7 +25873,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v5, s10
; VI-NEXT: v_mov_b32_e32 v16, s6
; VI-NEXT: v_mov_b32_e32 v17, s4
-; VI-NEXT: .LBB109_5: ; %end
+; VI-NEXT: .LBB109_6: ; %end
; VI-NEXT: v_mov_b32_e32 v3, v17
; VI-NEXT: v_mov_b32_e32 v1, v11
; VI-NEXT: v_mov_b32_e32 v11, v16
@@ -24695,7 +25883,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB109_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s19, 24
; GFX9-NEXT: s_lshr_b32 s22, s19, 16
@@ -24709,8 +25897,28 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s14, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB109_4
-; GFX9-NEXT: .LBB109_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB109_3
+; GFX9-NEXT: .LBB109_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr21
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr20
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr23
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr22
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB109_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -24800,22 +26008,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v18
-; GFX9-NEXT: s_branch .LBB109_5
-; GFX9-NEXT: .LBB109_3:
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr21
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr20
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr23
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr22
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB109_2
-; GFX9-NEXT: .LBB109_4:
+; GFX9-NEXT: s_branch .LBB109_6
+; GFX9-NEXT: .LBB109_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s18
; GFX9-NEXT: v_mov_b32_e32 v10, s23
; GFX9-NEXT: v_mov_b32_e32 v16, s19
@@ -24832,7 +26026,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v5, s11
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB109_5: ; %end
+; GFX9-NEXT: .LBB109_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v17
; GFX9-NEXT: v_mov_b32_e32 v12, v16
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -24842,7 +26036,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s3, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s3, 16
@@ -24856,9 +26050,28 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-TRUE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB109_3
+; GFX11-TRUE16-NEXT: .LBB109_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -24958,22 +26171,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
-; GFX11-TRUE16-NEXT: s_branch .LBB109_5
-; GFX11-TRUE16-NEXT: .LBB109_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: s_branch .LBB109_2
-; GFX11-TRUE16-NEXT: .LBB109_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
+; GFX11-TRUE16-NEXT: .LBB109_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v17, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v9, s14
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s3 :: v_dual_mov_b32 v15, s11
@@ -24983,7 +26182,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v5, s10
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v17
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v16
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -24993,7 +26192,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s3, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s17, s3, 16
@@ -25007,9 +26206,28 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s0, 8
; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-FAKE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB109_3
+; GFX11-FAKE16-NEXT: .LBB109_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s1, 0, s1
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -25105,22 +26323,8 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
-; GFX11-FAKE16-NEXT: s_branch .LBB109_5
-; GFX11-FAKE16-NEXT: .LBB109_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: s_branch .LBB109_2
-; GFX11-FAKE16-NEXT: .LBB109_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB109_6
+; GFX11-FAKE16-NEXT: .LBB109_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v17, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v9, s14
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, s3 :: v_dual_mov_b32 v15, s11
@@ -25130,7 +26334,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v5, s10
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-FAKE16-NEXT: .LBB109_5: ; %end
+; GFX11-FAKE16-NEXT: .LBB109_6: ; %end
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v17
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v16
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -25707,7 +26911,7 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB111_4
+; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -25741,8 +26945,24 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s12, 24
; SI-NEXT: s_or_b32 s15, s5, s4
-; SI-NEXT: s_cbranch_execnz .LBB111_3
-; SI-NEXT: .LBB111_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB111_3
+; SI-NEXT: .LBB111_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: .LBB111_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB111_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_and_b32 s4, s28, 0xff
; SI-NEXT: s_lshl_b32 s5, s29, 8
@@ -25803,7 +27023,7 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s9, s5, 16
; SI-NEXT: s_and_b32 s15, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s4, 16
-; SI-NEXT: .LBB111_3: ; %end
+; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
@@ -25821,16 +27041,6 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s11
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB111_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: s_branch .LBB111_2
;
; VI-LABEL: bitcast_v16i8_to_v8bf16_scalar:
; VI: ; %bb.0:
@@ -25839,7 +27049,7 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB111_4
+; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -25866,8 +27076,17 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s7, v5, v3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: s_cbranch_execnz .LBB111_3
-; VI-NEXT: .LBB111_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB111_3
+; VI-NEXT: .LBB111_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB111_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB111_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -25909,11 +27128,8 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x3000000, v3
-; VI-NEXT: .LBB111_3: ; %end
+; VI-NEXT: .LBB111_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB111_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB111_2
;
; GFX9-LABEL: bitcast_v16i8_to_v8bf16_scalar:
; GFX9: ; %bb.0:
@@ -25922,7 +27138,7 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -25949,8 +27165,17 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s7, v5, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX9-NEXT: s_cbranch_execnz .LBB111_3
-; GFX9-NEXT: .LBB111_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB111_3
+; GFX9-NEXT: .LBB111_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB111_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc0c0004
@@ -25989,18 +27214,15 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX9-NEXT: v_add_u32_e32 v4, 0x300, v4
; GFX9-NEXT: v_add_u32_sdwa v3, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB111_3: ; %end
+; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB111_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB111_2
;
; GFX11-LABEL: bitcast_v16i8_to_v8bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -26022,9 +27244,17 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB111_3
+; GFX11-NEXT: .LBB111_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB111_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -26063,11 +27293,8 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB111_3: ; %end
+; GFX11-NEXT: .LBB111_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB111_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB111_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
index c09389ef700ac..e34b6552733eb 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.160bit.ll
@@ -97,94 +97,115 @@ define inreg <5 x float> @bitcast_v5i32_to_v5f32_scalar(<5 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v5i32_to_v5f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v5i32_to_v5f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v5i32_to_v5f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_mov_b32_e32 v4, s16
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -291,19 +312,25 @@ define inreg <5 x i32> @bitcast_v5f32_to_v5i32_scalar(<5 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -315,19 +342,25 @@ define inreg <5 x i32> @bitcast_v5f32_to_v5i32_scalar(<5 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -339,19 +372,25 @@ define inreg <5 x i32> @bitcast_v5f32_to_v5i32_scalar(<5 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -365,19 +404,22 @@ define inreg <5 x i32> @bitcast_v5f32_to_v5i32_scalar(<5 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s5, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -524,15 +566,28 @@ define inreg <10 x i16> @bitcast_v5i32_to_v10i16_scalar(<5 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s19, 16
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -543,7 +598,7 @@ define inreg <10 x i16> @bitcast_v5i32_to_v10i16_scalar(<5 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s19, 16
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -565,83 +620,91 @@ define inreg <10 x i16> @bitcast_v5i32_to_v10i16_scalar(<5 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v4, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v5i32_to_v10i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v5i32_to_v10i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v5i32_to_v10i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_mov_b32_e32 v4, s16
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -827,7 +890,7 @@ define inreg <5 x i32> @bitcast_v10i16_to_v5i32_scalar(<10 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -844,8 +907,17 @@ define inreg <5 x i32> @bitcast_v10i16_to_v5i32_scalar(<10 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s8, s20, 0xffff
; SI-NEXT: s_lshl_b32 s10, s9, 16
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -871,25 +943,30 @@ define inreg <5 x i32> @bitcast_v10i16_to_v5i32_scalar(<10 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v10i16_to_v5i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s20, 3
; VI-NEXT: s_and_b32 s4, s20, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -915,33 +992,37 @@ define inreg <5 x i32> @bitcast_v10i16_to_v5i32_scalar(<10 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v10i16_to_v5i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -955,19 +1036,22 @@ define inreg <5 x i32> @bitcast_v10i16_to_v5i32_scalar(<10 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s5, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1114,15 +1198,28 @@ define inreg <10 x half> @bitcast_v5i32_to_v10f16_scalar(<5 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s19, 16
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -1133,7 +1230,7 @@ define inreg <10 x half> @bitcast_v5i32_to_v10f16_scalar(<5 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s19, 16
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1155,83 +1252,91 @@ define inreg <10 x half> @bitcast_v5i32_to_v10f16_scalar(<5 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v4, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v5i32_to_v10f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v5i32_to_v10f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v3, s19
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v5i32_to_v10f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_mov_b32_e32 v4, s16
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1437,7 +1542,7 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -1454,8 +1559,17 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s8, s20, 0xffff
; SI-NEXT: s_lshl_b32 s10, s9, 16
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
@@ -1497,10 +1611,7 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1512,10 +1623,18 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s20, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -1543,9 +1662,7 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1557,10 +1674,18 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
@@ -1568,9 +1693,7 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1584,19 +1707,22 @@ define inreg <5 x i32> @bitcast_v10f16_to_v5i32_scalar(<10 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s5, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1741,15 +1867,28 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_3
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s13, s19, 16
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_4
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -1760,15 +1899,8 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[5:6], v[4:5], 16
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: s_branch .LBB13_5
-; SI-NEXT: .LBB13_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB13_2
-; SI-NEXT: .LBB13_4:
+; SI-NEXT: s_branch .LBB13_6
+; SI-NEXT: .LBB13_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1779,7 +1911,7 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: .LBB13_5: ; %end
+; SI-NEXT: .LBB13_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v1, v1, v6
@@ -1801,19 +1933,25 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_3
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_4
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_3:
-; VI-NEXT: s_branch .LBB13_2
-; VI-NEXT: .LBB13_4:
+; VI-NEXT: .LBB13_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1828,19 +1966,25 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_4
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_3:
-; GFX9-NEXT: s_branch .LBB13_2
-; GFX9-NEXT: .LBB13_4:
+; GFX9-NEXT: .LBB13_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1857,19 +2001,22 @@ define inreg <10 x i16> @bitcast_v5f32_to_v10i16_scalar(<5 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_4
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_3:
-; GFX11-NEXT: s_branch .LBB13_2
; GFX11-NEXT: .LBB13_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2061,7 +2208,7 @@ define inreg <5 x float> @bitcast_v10i16_to_v5f32_scalar(<10 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -2078,8 +2225,17 @@ define inreg <5 x float> @bitcast_v10i16_to_v5f32_scalar(<10 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s8, s20, 0xffff
; SI-NEXT: s_lshl_b32 s10, s9, 16
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -2105,25 +2261,30 @@ define inreg <5 x float> @bitcast_v10i16_to_v5f32_scalar(<10 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v10i16_to_v5f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s20, 3
; VI-NEXT: s_and_b32 s4, s20, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2149,33 +2310,37 @@ define inreg <5 x float> @bitcast_v10i16_to_v5f32_scalar(<10 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v10i16_to_v5f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2189,19 +2354,22 @@ define inreg <5 x float> @bitcast_v10i16_to_v5f32_scalar(<10 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s5, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2346,15 +2514,28 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_3
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s13, s19, 16
; SI-NEXT: s_lshr_b32 s12, s17, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_4
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -2365,15 +2546,8 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; SI-NEXT: v_lshr_b64 v[5:6], v[4:5], 16
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; SI-NEXT: s_branch .LBB17_5
-; SI-NEXT: .LBB17_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB17_2
-; SI-NEXT: .LBB17_4:
+; SI-NEXT: s_branch .LBB17_6
+; SI-NEXT: .LBB17_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2384,7 +2558,7 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v5, s6
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: .LBB17_5: ; %end
+; SI-NEXT: .LBB17_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v1, v1, v6
@@ -2406,19 +2580,25 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_3
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_4
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_3:
-; VI-NEXT: s_branch .LBB17_2
-; VI-NEXT: .LBB17_4:
+; VI-NEXT: .LBB17_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2433,19 +2613,25 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_4
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_3:
-; GFX9-NEXT: s_branch .LBB17_2
-; GFX9-NEXT: .LBB17_4:
+; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2462,19 +2648,22 @@ define inreg <10 x half> @bitcast_v5f32_to_v10f16_scalar(<5 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_4
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_3:
-; GFX11-NEXT: s_branch .LBB17_2
; GFX11-NEXT: .LBB17_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2686,7 +2875,7 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -2703,8 +2892,17 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; SI-NEXT: s_and_b32 s8, s20, 0xffff
; SI-NEXT: s_lshl_b32 s10, s9, 16
; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
@@ -2746,10 +2944,7 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2761,10 +2956,18 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s20, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2792,9 +2995,7 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2806,10 +3007,18 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
@@ -2817,9 +3026,7 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2833,19 +3040,22 @@ define inreg <5 x float> @bitcast_v10f16_to_v5f32_scalar(<10 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s5, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3055,7 +3265,7 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s15, s17, 16
; SI-NEXT: s_lshr_b32 s23, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s20, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
@@ -3074,10 +3284,23 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s4, s4, s28
; SI-NEXT: s_lshr_b64 s[8:9], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[28:29], 16
+; SI-NEXT: s_mov_b64 s[12:13], 0
; SI-NEXT: s_mov_b32 s7, s27
; SI-NEXT: s_mov_b32 s5, s29
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_and_b32 s4, s18, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -3108,7 +3331,7 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s15, s7, 16
; SI-NEXT: s_lshr_b32 s22, s5, 16
; SI-NEXT: s_lshr_b32 s14, s21, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_or_b32 s6, s6, s8
@@ -3130,22 +3353,23 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v10i16_to_v10f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -3171,33 +3395,37 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v3, s19
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v10i16_to_v10f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3214,19 +3442,22 @@ define inreg <10 x half> @bitcast_v10i16_to_v10f16_scalar(<10 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3417,10 +3648,18 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s10, s16, 16
; SI-NEXT: s_cmp_lg_u32 s21, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s10
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s9
@@ -3463,10 +3702,8 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[4:5], v[2:3], 16
; SI-NEXT: v_or_b32_e32 v5, v14, v13
-; SI-NEXT: s_branch .LBB23_5
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v11, s7
; SI-NEXT: v_mov_b32_e32 v12, s6
@@ -3477,7 +3714,7 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v9, s16
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v4, s9
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: .LBB23_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v9
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -3499,10 +3736,18 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s21, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -3530,9 +3775,7 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v7
; VI-NEXT: v_or_b32_e32 v0, v5, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3547,10 +3790,18 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s21, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v3, s19, v0 op_sel_hi:[1,0]
@@ -3558,9 +3809,7 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3577,19 +3826,22 @@ define inreg <10 x i16> @bitcast_v10f16_to_v10i16_scalar(<10 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s17, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index 30193251552ae..6588f7e82ed32 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -105,62 +105,82 @@ define inreg half @bitcast_i16_to_f16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_and_b32 s6, s16, 0xffff
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_i16_to_f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_i16_to_f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_i16_to_f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB1_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_3:
-; GFX11-NEXT: .LBB1_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -292,19 +312,25 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s16, 0xffff
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -312,16 +338,22 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -329,16 +361,22 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_add_f16_e32 v0, s16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -347,15 +385,18 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-TRUE16-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB3_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB3_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB3_2
; GFX11-TRUE16-NEXT: .LBB3_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -365,15 +406,18 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-FAKE16-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB3_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, 0x200, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB3_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB3_2
; GFX11-FAKE16-NEXT: .LBB3_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -496,66 +540,86 @@ define inreg bfloat @bitcast_i16_to_bf16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_and_b32 s6, s16, 0xffff
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s7, s6, 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_lshl_b32 s4, s6, 16
; SI-NEXT: s_add_i32 s7, s4, 0x30000
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_i16_to_bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_i16_to_bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_i16_to_bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB5_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_3:
-; GFX11-NEXT: .LBB5_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -726,28 +790,42 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; SI-NEXT: s_lshl_b32 s4, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_bf16_to_i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_3
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_4
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v0
@@ -759,9 +837,7 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_3:
-; VI-NEXT: s_branch .LBB7_2
-; VI-NEXT: .LBB7_4:
+; VI-NEXT: .LBB7_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -769,10 +845,18 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
@@ -784,9 +868,7 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -795,11 +877,16 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -813,8 +900,6 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -943,22 +1028,28 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_3
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB9_4
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB9_5
-; SI-NEXT: .LBB9_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB9_2
-; SI-NEXT: .LBB9_4:
+; SI-NEXT: s_branch .LBB9_6
+; SI-NEXT: .LBB9_5:
; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: .LBB9_5: ; %end
+; SI-NEXT: .LBB9_6: ; %end
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -967,16 +1058,22 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_3
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_4
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_3:
-; VI-NEXT: s_branch .LBB9_2
-; VI-NEXT: .LBB9_4:
+; VI-NEXT: .LBB9_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -984,16 +1081,22 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_4
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_add_f16_e32 v0, s16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_3:
-; GFX9-NEXT: s_branch .LBB9_2
-; GFX9-NEXT: .LBB9_4:
+; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -1002,15 +1105,18 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB9_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB9_4
-; GFX11-TRUE16-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB9_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB9_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB9_2
; GFX11-TRUE16-NEXT: .LBB9_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1020,15 +1126,18 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB9_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB9_4
-; GFX11-FAKE16-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB9_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, 0x200, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB9_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB9_2
; GFX11-FAKE16-NEXT: .LBB9_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1199,28 +1308,42 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; SI-NEXT: s_lshl_b32 s4, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB11_4
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; SI-NEXT: s_cbranch_execnz .LBB11_3
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: .LBB11_3: ; %end
+; SI-NEXT: .LBB11_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_branch .LBB11_2
;
; VI-LABEL: bitcast_bf16_to_f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v0
@@ -1232,9 +1355,7 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -1242,10 +1363,18 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_lshl_b32 s4, s16, 16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v0
@@ -1257,9 +1386,7 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -1268,11 +1395,16 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -1286,8 +1418,6 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
index 647b212d4d0bf..29f303ceeabb5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.192bit.ll
@@ -101,17 +101,25 @@ define inreg <6 x float> @bitcast_v6i32_to_v6f32_scalar(<6 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -119,24 +127,30 @@ define inreg <6 x float> @bitcast_v6i32_to_v6f32_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v6i32_to_v6f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -144,24 +158,30 @@ define inreg <6 x float> @bitcast_v6i32_to_v6f32_scalar(<6 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v6i32_to_v6f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -169,33 +189,34 @@ define inreg <6 x float> @bitcast_v6i32_to_v6f32_scalar(<6 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v6i32_to_v6f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -305,10 +326,18 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -316,9 +345,7 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -331,10 +358,18 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -342,9 +377,7 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -357,10 +390,18 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -368,9 +409,7 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -386,11 +425,16 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
@@ -398,8 +442,6 @@ define inreg <6 x i32> @bitcast_v6f32_to_v6i32_scalar(<6 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -517,17 +559,25 @@ define inreg <3 x i64> @bitcast_v6i32_to_v3i64_scalar(<6 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -535,24 +585,30 @@ define inreg <3 x i64> @bitcast_v6i32_to_v3i64_scalar(<6 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v6i32_to_v3i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -560,24 +616,30 @@ define inreg <3 x i64> @bitcast_v6i32_to_v3i64_scalar(<6 x i32> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v6i32_to_v3i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -585,33 +647,34 @@ define inreg <3 x i64> @bitcast_v6i32_to_v3i64_scalar(<6 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v6i32_to_v3i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -726,17 +789,25 @@ define inreg <6 x i32> @bitcast_v3i64_to_v6i32_scalar(<3 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s20, s20, 3
; SI-NEXT: s_addc_u32 s21, s21, 0
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -744,24 +815,30 @@ define inreg <6 x i32> @bitcast_v3i64_to_v6i32_scalar(<3 x i64> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v3i64_to_v6i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s20, s20, 3
; VI-NEXT: s_addc_u32 s21, s21, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -769,24 +846,30 @@ define inreg <6 x i32> @bitcast_v3i64_to_v6i32_scalar(<3 x i64> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v3i64_to_v6i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s20, s20, 3
; GFX9-NEXT: s_addc_u32 s21, s21, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -794,33 +877,34 @@ define inreg <6 x i32> @bitcast_v3i64_to_v6i32_scalar(<3 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v3i64_to_v6i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s16, s16, 3
; GFX11-NEXT: s_addc_u32 s17, s17, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -933,17 +1017,25 @@ define inreg <3 x double> @bitcast_v6i32_to_v3f64_scalar(<6 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -951,24 +1043,30 @@ define inreg <3 x double> @bitcast_v6i32_to_v3f64_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v6i32_to_v3f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -976,24 +1074,30 @@ define inreg <3 x double> @bitcast_v6i32_to_v3f64_scalar(<6 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v6i32_to_v3f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1001,33 +1105,34 @@ define inreg <3 x double> @bitcast_v6i32_to_v3f64_scalar(<6 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v6i32_to_v3f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1131,17 +1236,23 @@ define inreg <6 x i32> @bitcast_v3f64_to_v6i32_scalar(<3 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1154,17 +1265,23 @@ define inreg <6 x i32> @bitcast_v3f64_to_v6i32_scalar(<3 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1177,17 +1294,23 @@ define inreg <6 x i32> @bitcast_v3f64_to_v6i32_scalar(<3 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1203,17 +1326,20 @@ define inreg <6 x i32> @bitcast_v3f64_to_v6i32_scalar(<3 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1370,7 +1496,7 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -1378,8 +1504,22 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1392,7 +1532,7 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
; SI-NEXT: s_lshr_b32 s14, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1418,30 +1558,30 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v6i32_to_v12i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1449,24 +1589,30 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v6i32_to_v12i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1474,33 +1620,34 @@ define inreg <12 x i16> @bitcast_v6i32_to_v12i16_scalar(<6 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v6i32_to_v12i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1703,7 +1850,7 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -1723,8 +1870,17 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -1755,7 +1911,7 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1763,18 +1919,23 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v12i16_to_v6i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s21, 3
; VI-NEXT: s_and_b32 s4, s21, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1805,7 +1966,7 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1813,17 +1974,23 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v12i16_to_v6i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
@@ -1831,9 +1998,7 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1849,11 +2014,16 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
@@ -1861,8 +2031,6 @@ define inreg <6 x i32> @bitcast_v12i16_to_v6i32_scalar(<12 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2019,7 +2187,7 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -2027,8 +2195,22 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -2041,7 +2223,7 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
; SI-NEXT: s_lshr_b32 s14, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -2067,30 +2249,30 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v6i32_to_v12f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2098,24 +2280,30 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v6i32_to_v12f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2123,33 +2311,34 @@ define inreg <12 x half> @bitcast_v6i32_to_v12f16_scalar(<6 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v6i32_to_v12f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2376,7 +2565,7 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -2396,8 +2585,17 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
@@ -2447,10 +2645,7 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2463,10 +2658,18 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s21, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2499,9 +2702,7 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2514,10 +2715,18 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
@@ -2526,9 +2735,7 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2544,11 +2751,16 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
@@ -2556,8 +2768,6 @@ define inreg <6 x i32> @bitcast_v12f16_to_v6i32_scalar(<12 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2672,10 +2882,18 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -2683,9 +2901,7 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2698,10 +2914,18 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -2709,9 +2933,7 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2724,10 +2946,18 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -2735,9 +2965,7 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2753,11 +2981,16 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
@@ -2765,8 +2998,6 @@ define inreg <3 x i64> @bitcast_v6f32_to_v3i64_scalar(<6 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2886,17 +3117,25 @@ define inreg <6 x float> @bitcast_v3i64_to_v6f32_scalar(<3 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s20, s20, 3
; SI-NEXT: s_addc_u32 s21, s21, 0
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2904,24 +3143,30 @@ define inreg <6 x float> @bitcast_v3i64_to_v6f32_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v3i64_to_v6f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s20, s20, 3
; VI-NEXT: s_addc_u32 s21, s21, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2929,24 +3174,30 @@ define inreg <6 x float> @bitcast_v3i64_to_v6f32_scalar(<3 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v3i64_to_v6f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s20, s20, 3
; GFX9-NEXT: s_addc_u32 s21, s21, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2954,33 +3205,34 @@ define inreg <6 x float> @bitcast_v3i64_to_v6f32_scalar(<3 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v3i64_to_v6f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s16, s16, 3
; GFX11-NEXT: s_addc_u32 s17, s17, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3090,10 +3342,18 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3101,9 +3361,7 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3116,10 +3374,18 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3127,9 +3393,7 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3142,10 +3406,18 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3153,9 +3425,7 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3171,11 +3441,16 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
@@ -3183,8 +3458,6 @@ define inreg <3 x double> @bitcast_v6f32_to_v3f64_scalar(<6 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3293,17 +3566,23 @@ define inreg <6 x float> @bitcast_v3f64_to_v6f32_scalar(<3 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3316,17 +3595,23 @@ define inreg <6 x float> @bitcast_v3f64_to_v6f32_scalar(<3 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3339,17 +3624,23 @@ define inreg <6 x float> @bitcast_v3f64_to_v6f32_scalar(<3 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3365,17 +3656,20 @@ define inreg <6 x float> @bitcast_v3f64_to_v6f32_scalar(<3 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3529,7 +3823,7 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -3537,8 +3831,22 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3551,16 +3859,8 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3573,7 +3873,7 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s6
; SI-NEXT: v_mov_b32_e32 v6, s4
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -3598,10 +3898,18 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3609,9 +3917,7 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3626,10 +3932,18 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -3637,9 +3951,7 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3657,11 +3969,16 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
@@ -3669,8 +3986,6 @@ define inreg <12 x i16> @bitcast_v6f32_to_v12i16_scalar(<6 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3879,7 +4194,7 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -3899,8 +4214,17 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -3931,7 +4255,7 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -3939,18 +4263,23 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v12i16_to_v6f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s21, 3
; VI-NEXT: s_and_b32 s4, s21, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -3981,7 +4310,7 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3989,17 +4318,23 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v12i16_to_v6f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
@@ -4007,9 +4342,7 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4025,11 +4358,16 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
@@ -4037,8 +4375,6 @@ define inreg <6 x float> @bitcast_v12i16_to_v6f32_scalar(<12 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4192,7 +4528,7 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -4200,8 +4536,22 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -4214,16 +4564,8 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4236,7 +4578,7 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s6
; SI-NEXT: v_mov_b32_e32 v6, s4
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -4261,10 +4603,18 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
; VI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -4272,9 +4622,7 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4289,10 +4637,18 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
; GFX9-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -4300,9 +4656,7 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4320,11 +4674,16 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s3, 1.0
@@ -4332,8 +4691,6 @@ define inreg <12 x half> @bitcast_v6f32_to_v12f16_scalar(<6 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4566,7 +4923,7 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -4586,8 +4943,17 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
@@ -4637,10 +5003,7 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -4653,10 +5016,18 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s21, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -4689,9 +5060,7 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4704,10 +5073,18 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
@@ -4716,9 +5093,7 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4734,11 +5109,16 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
@@ -4746,8 +5126,6 @@ define inreg <6 x float> @bitcast_v12f16_to_v6f32_scalar(<12 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4867,17 +5245,25 @@ define inreg <3 x double> @bitcast_v3i64_to_v3f64_scalar(<3 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s20, s20, 3
; SI-NEXT: s_addc_u32 s21, s21, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4885,24 +5271,30 @@ define inreg <3 x double> @bitcast_v3i64_to_v3f64_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s20
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v3i64_to_v3f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s20, s20, 3
; VI-NEXT: s_addc_u32 s21, s21, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4910,24 +5302,30 @@ define inreg <3 x double> @bitcast_v3i64_to_v3f64_scalar(<3 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v3i64_to_v3f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
; GFX9-NEXT: s_addc_u32 s21, s21, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4935,32 +5333,33 @@ define inreg <3 x double> @bitcast_v3i64_to_v3f64_scalar(<3 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v3i64_to_v3f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
; GFX11-NEXT: s_addc_u32 s17, s17, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5064,17 +5463,23 @@ define inreg <3 x i64> @bitcast_v3f64_to_v3i64_scalar(<3 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5087,17 +5492,23 @@ define inreg <3 x i64> @bitcast_v3f64_to_v3i64_scalar(<3 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5110,17 +5521,23 @@ define inreg <3 x i64> @bitcast_v3f64_to_v3i64_scalar(<3 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5136,17 +5553,20 @@ define inreg <3 x i64> @bitcast_v3f64_to_v3i64_scalar(<3 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5305,7 +5725,7 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -5313,8 +5733,22 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s20, s20, 3
; SI-NEXT: s_addc_u32 s21, s21, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -5327,7 +5761,7 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -5353,30 +5787,30 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v3i64_to_v12i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s20, s20, 3
; VI-NEXT: s_addc_u32 s21, s21, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5384,24 +5818,30 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v3i64_to_v12i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s20, s20, 3
; GFX9-NEXT: s_addc_u32 s21, s21, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5409,33 +5849,34 @@ define inreg <12 x i16> @bitcast_v3i64_to_v12i16_scalar(<3 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v3i64_to_v12i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s16, s16, 3
; GFX11-NEXT: s_addc_u32 s17, s17, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5638,7 +6079,7 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -5658,8 +6099,17 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -5690,7 +6140,7 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -5698,18 +6148,23 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v12i16_to_v3i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s21, 3
; VI-NEXT: s_and_b32 s4, s21, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -5740,7 +6195,7 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5748,17 +6203,23 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v12i16_to_v3i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
@@ -5766,9 +6227,7 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5784,11 +6243,16 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
@@ -5796,8 +6260,6 @@ define inreg <3 x i64> @bitcast_v12i16_to_v3i64_scalar(<12 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5956,7 +6418,7 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -5964,8 +6426,22 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s20, s20, 3
; SI-NEXT: s_addc_u32 s21, s21, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -5978,7 +6454,7 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -6004,30 +6480,30 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v3i64_to_v12f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s20, s20, 3
; VI-NEXT: s_addc_u32 s21, s21, 0
; VI-NEXT: s_add_u32 s18, s18, 3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6035,24 +6511,30 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v3i64_to_v12f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s20, s20, 3
; GFX9-NEXT: s_addc_u32 s21, s21, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6060,33 +6542,34 @@ define inreg <12 x half> @bitcast_v3i64_to_v12f16_scalar(<3 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s20
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v3i64_to_v12f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s16, s16, 3
; GFX11-NEXT: s_addc_u32 s17, s17, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6313,7 +6796,7 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -6333,8 +6816,17 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
@@ -6384,10 +6876,7 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -6400,10 +6889,18 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s21, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -6436,9 +6933,7 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6451,10 +6946,18 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
@@ -6463,9 +6966,7 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6481,11 +6982,16 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
@@ -6493,8 +6999,6 @@ define inreg <3 x i64> @bitcast_v12f16_to_v3i64_scalar(<12 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6642,7 +7146,7 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -6650,8 +7154,22 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
@@ -6661,16 +7179,8 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v4, s20
@@ -6683,7 +7193,7 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s6
; SI-NEXT: v_mov_b32_e32 v6, s4
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -6708,17 +7218,23 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6733,17 +7249,23 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6761,17 +7283,20 @@ define inreg <12 x i16> @bitcast_v3f64_to_v12i16_scalar(<3 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB49_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: s_branch .LBB49_2
; GFX11-NEXT: .LBB49_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6980,7 +7505,7 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -7000,8 +7525,17 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -7032,7 +7566,7 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -7040,18 +7574,23 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v12i16_to_v3f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s21, 3
; VI-NEXT: s_and_b32 s4, s21, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -7082,7 +7621,7 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7090,17 +7629,23 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v12i16_to_v3f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
@@ -7108,9 +7653,7 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7126,11 +7669,16 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
@@ -7138,8 +7686,6 @@ define inreg <3 x double> @bitcast_v12i16_to_v3f64_scalar(<12 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7287,7 +7833,7 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s13, s19, 16
@@ -7295,8 +7841,22 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
@@ -7306,16 +7866,8 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v4, s20
@@ -7328,7 +7880,7 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: v_mov_b32_e32 v7, s6
; SI-NEXT: v_mov_b32_e32 v6, s4
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -7353,17 +7905,23 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7378,17 +7936,23 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7406,17 +7970,20 @@ define inreg <12 x half> @bitcast_v3f64_to_v12f16_scalar(<3 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_3:
-; GFX11-NEXT: s_branch .LBB53_2
; GFX11-NEXT: .LBB53_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7649,7 +8216,7 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s24, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s24, 16
@@ -7669,8 +8236,17 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; SI-NEXT: s_and_b32 s9, s21, 0xffff
; SI-NEXT: s_lshl_b32 s10, s12, 16
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s10, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s24
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s23
@@ -7720,10 +8296,7 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -7736,10 +8309,18 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s21, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -7772,9 +8353,7 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7787,10 +8366,18 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
@@ -7799,9 +8386,7 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7817,11 +8402,16 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
@@ -7829,8 +8419,6 @@ define inreg <3 x double> @bitcast_v12f16_to_v3f64_scalar(<12 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8063,7 +8651,7 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s24, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s24, 16
@@ -8086,11 +8674,25 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[10:11], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[42:43], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[44:45], 16
+; SI-NEXT: s_mov_b64 s[22:23], 0
; SI-NEXT: s_mov_b32 s9, s41
; SI-NEXT: s_mov_b32 s7, s43
; SI-NEXT: s_mov_b32 s5, s45
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[22:23], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[22:23], s[22:23], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s29, 16
@@ -8127,7 +8729,7 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s24, s9, 16
; SI-NEXT: s_lshr_b32 s25, s7, 16
; SI-NEXT: s_lshr_b32 s26, s5, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_or_b32 s8, s8, s10
@@ -8153,23 +8755,23 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v12i16_to_v12f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -8200,7 +8802,7 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8208,17 +8810,23 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v4, s20
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v12i16_to_v12f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v3, s19, 3 op_sel_hi:[1,0]
@@ -8226,9 +8834,7 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8246,11 +8852,16 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s3, 3 op_sel_hi:[1,0]
@@ -8258,8 +8869,6 @@ define inreg <12 x half> @bitcast_v12i16_to_v12f16_scalar(<12 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: s_branch .LBB57_2
; GFX11-NEXT: .LBB57_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8469,10 +9078,18 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s22, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s11
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s10
@@ -8524,10 +9141,8 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[8:9], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[6:7], v[4:5], 16
; SI-NEXT: v_or_b32_e32 v4, v17, v4
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v13, s8
; SI-NEXT: v_mov_b32_e32 v15, s7
; SI-NEXT: v_mov_b32_e32 v16, s6
@@ -8540,7 +9155,7 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s11
; SI-NEXT: v_mov_b32_e32 v8, s10
; SI-NEXT: v_mov_b32_e32 v6, s9
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v12
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v10
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -8565,10 +9180,18 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s22, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -8601,9 +9224,7 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v8
; VI-NEXT: v_or_b32_e32 v0, v6, v7
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8618,10 +9239,18 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s22, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v4, s20, v0 op_sel_hi:[1,0]
@@ -8630,9 +9259,7 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8650,11 +9277,16 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s18, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s3 op_sel_hi:[0,1]
@@ -8662,8 +9294,6 @@ define inreg <12 x i16> @bitcast_v12f16_to_v12i16_scalar(<12 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
index c66f5ea8e6c60..d217850515f07 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.224bit.ll
@@ -106,10 +106,18 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s20, s20, 3
@@ -117,7 +125,7 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -126,17 +134,23 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v7i32_to_v7f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
@@ -144,7 +158,7 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -153,17 +167,23 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v7i32_to_v7f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
@@ -171,7 +191,7 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -180,19 +200,22 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v7i32_to_v7f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
@@ -200,15 +223,13 @@ define inreg <7 x float> @bitcast_v7i32_to_v7f32_scalar(<7 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_mov_b32_e32 v6, s18
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -322,10 +343,18 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -334,9 +363,7 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -350,10 +377,18 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -362,9 +397,7 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -378,10 +411,18 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -390,9 +431,7 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -410,11 +449,16 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s7, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s7, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
@@ -423,8 +467,6 @@ define inreg <7 x i32> @bitcast_v7f32_to_v7i32_scalar(<7 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -593,7 +635,7 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s14, s21, 16
@@ -602,8 +644,23 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -618,7 +675,7 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s15, s19, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[22:23], 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -648,24 +705,23 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v5, s10
; SI-NEXT: v_mov_b32_e32 v6, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v7i32_to_v14i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
@@ -673,7 +729,7 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -682,17 +738,23 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v7i32_to_v14i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
@@ -700,7 +762,7 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -709,19 +771,22 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v7i32_to_v14i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
@@ -729,15 +794,13 @@ define inreg <14 x i16> @bitcast_v7i32_to_v14i16_scalar(<7 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_mov_b32_e32 v6, s18
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -958,7 +1021,7 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -981,8 +1044,17 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s10, s22, 0xffff
; SI-NEXT: s_lshl_b32 s12, s11, 16
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -1018,7 +1090,7 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1027,18 +1099,23 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v14i16_to_v7i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s22, 3
; VI-NEXT: s_and_b32 s4, s22, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1074,7 +1151,7 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1083,17 +1160,23 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v14i16_to_v7i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
@@ -1102,9 +1185,7 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1122,11 +1203,16 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s7, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s7, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
@@ -1135,8 +1221,6 @@ define inreg <7 x i32> @bitcast_v14i16_to_v7i32_scalar(<14 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1305,7 +1389,7 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s23, s17, 16
; SI-NEXT: s_lshr_b32 s14, s21, 16
@@ -1314,8 +1398,23 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -1330,7 +1429,7 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s21, 16
; SI-NEXT: s_lshr_b32 s15, s19, 16
; SI-NEXT: s_lshr_b64 s[6:7], s[22:23], 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1360,24 +1459,23 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v5, s10
; SI-NEXT: v_mov_b32_e32 v6, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v7i32_to_v14f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s20, s20, 3
@@ -1385,7 +1483,7 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1394,17 +1492,23 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v7i32_to_v14f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
; GFX9-NEXT: s_add_i32 s20, s20, 3
@@ -1412,7 +1516,7 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1421,19 +1525,22 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v5, s21
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v7i32_to_v14f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
; GFX11-NEXT: s_add_i32 s16, s16, 3
@@ -1441,15 +1548,13 @@ define inreg <14 x half> @bitcast_v7i32_to_v14f16_scalar(<7 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_mov_b32_e32 v6, s18
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1698,7 +1803,7 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -1721,8 +1826,17 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s10, s22, 0xffff
; SI-NEXT: s_lshl_b32 s12, s11, 16
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s27
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s26
@@ -1780,10 +1894,7 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1797,10 +1908,18 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s22, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -1838,9 +1957,7 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v7
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1854,10 +1971,18 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
@@ -1867,9 +1992,7 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1887,11 +2010,16 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s7, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s7, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
@@ -1900,8 +2028,6 @@ define inreg <7 x i32> @bitcast_v14f16_to_v7i32_scalar(<14 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2066,7 +2192,7 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_3
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s23, s21, 16
; SI-NEXT: s_lshr_b32 s15, s19, 16
@@ -2075,8 +2201,23 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_4
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -2091,17 +2232,8 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: s_branch .LBB13_5
-; SI-NEXT: .LBB13_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB13_2
-; SI-NEXT: .LBB13_4:
+; SI-NEXT: s_branch .LBB13_6
+; SI-NEXT: .LBB13_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2116,7 +2248,7 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v11, s10
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: .LBB13_5: ; %end
+; SI-NEXT: .LBB13_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
@@ -2144,10 +2276,18 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_3
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_4
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -2156,9 +2296,7 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_3:
-; VI-NEXT: s_branch .LBB13_2
-; VI-NEXT: .LBB13_4:
+; VI-NEXT: .LBB13_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2173,10 +2311,18 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_4
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -2185,9 +2331,7 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_3:
-; GFX9-NEXT: s_branch .LBB13_2
-; GFX9-NEXT: .LBB13_4:
+; GFX9-NEXT: .LBB13_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2206,11 +2350,16 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_4
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
@@ -2219,8 +2368,6 @@ define inreg <14 x i16> @bitcast_v7f32_to_v14i16_scalar(<7 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_3:
-; GFX11-NEXT: s_branch .LBB13_2
; GFX11-NEXT: .LBB13_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2447,7 +2594,7 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -2470,8 +2617,17 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s10, s22, 0xffff
; SI-NEXT: s_lshl_b32 s12, s11, 16
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -2507,7 +2663,7 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2516,18 +2672,23 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v14i16_to_v7f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s22, 3
; VI-NEXT: s_and_b32 s4, s22, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2563,7 +2724,7 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2572,17 +2733,23 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v14i16_to_v7f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
@@ -2591,9 +2758,7 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2611,11 +2776,16 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s7, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s7, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
@@ -2624,8 +2794,6 @@ define inreg <7 x float> @bitcast_v14i16_to_v7f32_scalar(<14 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2790,7 +2958,7 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_3
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s23, s21, 16
; SI-NEXT: s_lshr_b32 s15, s19, 16
@@ -2799,8 +2967,23 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; SI-NEXT: s_lshr_b64 s[4:5], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_4
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v3, s19, 1.0
@@ -2815,17 +2998,8 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: s_branch .LBB17_5
-; SI-NEXT: .LBB17_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB17_2
-; SI-NEXT: .LBB17_4:
+; SI-NEXT: s_branch .LBB17_6
+; SI-NEXT: .LBB17_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2840,7 +3014,7 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v11, s10
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: .LBB17_5: ; %end
+; SI-NEXT: .LBB17_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
@@ -2868,10 +3042,18 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_3
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_4
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
; VI-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -2880,9 +3062,7 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_3:
-; VI-NEXT: s_branch .LBB17_2
-; VI-NEXT: .LBB17_4:
+; VI-NEXT: .LBB17_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2897,10 +3077,18 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_4
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s20, 1.0
@@ -2909,9 +3097,7 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_3:
-; GFX9-NEXT: s_branch .LBB17_2
-; GFX9-NEXT: .LBB17_4:
+; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2930,11 +3116,16 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_4
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s4, 1.0
@@ -2943,8 +3134,6 @@ define inreg <14 x half> @bitcast_v7f32_to_v14f16_scalar(<7 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_3:
-; GFX11-NEXT: s_branch .LBB17_2
; GFX11-NEXT: .LBB17_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3199,7 +3388,7 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s26, s17, 16
; SI-NEXT: s_lshr_b32 s27, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s27, 16
@@ -3222,8 +3411,17 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; SI-NEXT: s_and_b32 s10, s22, 0xffff
; SI-NEXT: s_lshl_b32 s12, s11, 16
; SI-NEXT: s_or_b32 s10, s10, s12
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s27
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s26
@@ -3281,10 +3479,7 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -3298,10 +3493,18 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s22, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -3339,9 +3542,7 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v7
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3355,10 +3556,18 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
@@ -3368,9 +3577,7 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3388,11 +3595,16 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s7, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s7, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX11-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s7, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
@@ -3401,8 +3613,6 @@ define inreg <7 x float> @bitcast_v14f16_to_v7f32_scalar(<14 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3658,7 +3868,7 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s27, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s22, 0xffff
; SI-NEXT: s_lshl_b32 s7, s26, 16
@@ -3684,11 +3894,26 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[10:11], s[44:45], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[46:47], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[56:57], 16
+; SI-NEXT: s_mov_b64 s[24:25], 0
; SI-NEXT: s_mov_b32 s9, s45
; SI-NEXT: s_mov_b32 s7, s47
; SI-NEXT: s_mov_b32 s5, s57
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[24:25], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[24:25], s[24:25], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s42, 16
@@ -3731,7 +3956,7 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s28, s7, 16
; SI-NEXT: s_lshr_b32 s29, s5, 16
; SI-NEXT: s_lshr_b32 s26, s23, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_or_b32 s8, s8, s10
@@ -3761,24 +3986,23 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v5, s5
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v14i16_to_v14f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_add_i32 s7, s17, 3
@@ -3814,7 +4038,7 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3823,17 +4047,23 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v5, s21
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v14i16_to_v14f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s20, 3 op_sel_hi:[1,0]
@@ -3842,9 +4072,7 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3863,11 +4091,16 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s4, 3 op_sel_hi:[1,0]
@@ -3876,8 +4109,6 @@ define inreg <14 x half> @bitcast_v14i16_to_v14f16_scalar(<14 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4106,10 +4337,18 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s12, s16, 16
; SI-NEXT: s_cmp_lg_u32 s23, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s12
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s11
@@ -4169,10 +4408,8 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[8:9], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[6:7], v[4:5], 16
; SI-NEXT: v_or_b32_e32 v4, v19, v4
-; SI-NEXT: s_branch .LBB23_5
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v12, s10
; SI-NEXT: v_mov_b32_e32 v15, s8
; SI-NEXT: v_mov_b32_e32 v17, s7
@@ -4187,7 +4424,7 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s12
; SI-NEXT: v_mov_b32_e32 v8, s11
; SI-NEXT: v_mov_b32_e32 v6, s9
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: .LBB23_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v13
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v10
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -4215,10 +4452,18 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s23, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -4256,9 +4501,7 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v9
; VI-NEXT: v_or_b32_e32 v0, v7, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4273,10 +4516,18 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s23, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v5, s21, v0 op_sel_hi:[1,0]
@@ -4286,9 +4537,7 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4307,11 +4556,16 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s19, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s4 op_sel_hi:[0,1]
@@ -4320,8 +4574,6 @@ define inreg <14 x i16> @bitcast_v14f16_to_v14i16_scalar(<14 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 224b73353c3bc..fb4faa32cb2c7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -110,10 +110,18 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s21, s21, 3
@@ -122,7 +130,7 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -132,17 +140,23 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v8i32_to_v8f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -151,7 +165,7 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -161,17 +175,23 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v8i32_to_v8f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -180,7 +200,7 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -190,19 +210,22 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v8i32_to_v8f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -211,15 +234,13 @@ define inreg <8 x float> @bitcast_v8i32_to_v8f32_scalar(<8 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -336,10 +357,18 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -349,9 +378,7 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -366,10 +393,18 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -379,9 +414,7 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -396,10 +429,18 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -409,9 +450,7 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -431,11 +470,16 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -445,8 +489,6 @@ define inreg <8 x i32> @bitcast_v8f32_to_v8i32_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -574,10 +616,18 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s21, s21, 3
@@ -586,7 +636,7 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -596,17 +646,23 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v8i32_to_v4i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -615,7 +671,7 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -625,17 +681,23 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v8i32_to_v4i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -644,7 +706,7 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -654,19 +716,22 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v8i32_to_v4i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -675,15 +740,13 @@ define inreg <4 x i64> @bitcast_v8i32_to_v4i64_scalar(<8 x i32> inreg %a, i32 in
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -807,10 +870,18 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s20, s20, 3
@@ -819,7 +890,7 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -829,17 +900,23 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v4i64_to_v8i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s20, s20, 3
@@ -848,7 +925,7 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -858,17 +935,23 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v4i64_to_v8i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
@@ -877,7 +960,7 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -887,19 +970,22 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v4i64_to_v8i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
@@ -908,15 +994,13 @@ define inreg <8 x i32> @bitcast_v4i64_to_v8i32_scalar(<4 x i64> inreg %a, i32 in
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1038,10 +1122,18 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s21, s21, 3
@@ -1050,7 +1142,7 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1060,17 +1152,23 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v8i32_to_v4f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -1079,7 +1177,7 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1089,17 +1187,23 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v8i32_to_v4f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -1108,7 +1212,7 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1118,19 +1222,22 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v8i32_to_v4f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -1139,15 +1246,13 @@ define inreg <4 x double> @bitcast_v8i32_to_v4f64_scalar(<8 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1255,18 +1360,24 @@ define inreg <8 x i32> @bitcast_v4f64_to_v8i32_scalar(<4 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1281,18 +1392,24 @@ define inreg <8 x i32> @bitcast_v4f64_to_v8i32_scalar(<4 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1307,18 +1424,24 @@ define inreg <8 x i32> @bitcast_v4f64_to_v8i32_scalar(<4 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1338,18 +1461,21 @@ define inreg <8 x i32> @bitcast_v4f64_to_v8i32_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1528,7 +1654,7 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s23, 16
; SI-NEXT: s_lshr_b32 s15, s21, 16
@@ -1538,8 +1664,24 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1556,7 +1698,7 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s15, s21, 16
; SI-NEXT: s_lshr_b32 s24, s19, 16
; SI-NEXT: s_lshr_b32 s25, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1590,25 +1732,23 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v8i32_to_v16i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -1617,7 +1757,7 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1627,17 +1767,23 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v8i32_to_v16i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -1646,7 +1792,7 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1656,19 +1802,22 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v8i32_to_v16i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -1677,15 +1826,13 @@ define inreg <16 x i16> @bitcast_v8i32_to_v16i16_scalar(<8 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1923,7 +2070,7 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -1949,8 +2096,17 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -1991,7 +2147,7 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2001,18 +2157,23 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v16i16_to_v8i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s23, 3
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2053,7 +2214,7 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2063,17 +2224,23 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v16i16_to_v8i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -2083,9 +2250,7 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2105,11 +2270,16 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -2119,8 +2289,6 @@ define inreg <8 x i32> @bitcast_v16i16_to_v8i32_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2299,7 +2467,7 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s23, 16
; SI-NEXT: s_lshr_b32 s15, s21, 16
@@ -2309,8 +2477,24 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -2327,7 +2511,7 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s15, s21, 16
; SI-NEXT: s_lshr_b32 s24, s19, 16
; SI-NEXT: s_lshr_b32 s25, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -2361,25 +2545,23 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v8i32_to_v16f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -2388,7 +2570,7 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2398,17 +2580,23 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v8i32_to_v16f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -2417,7 +2605,7 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2427,19 +2615,22 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v8i32_to_v16f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -2448,15 +2639,13 @@ define inreg <16 x half> @bitcast_v8i32_to_v16f16_scalar(<8 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2726,7 +2915,7 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -2752,8 +2941,17 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s40
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -2819,10 +3017,7 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2837,10 +3032,18 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2883,9 +3086,7 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2900,10 +3101,18 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -2914,9 +3123,7 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2936,11 +3143,16 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -2950,8 +3162,6 @@ define inreg <8 x i32> @bitcast_v16f16_to_v8i32_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3171,7 +3381,7 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s23, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s23, 16
@@ -3189,8 +3399,32 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; SI-NEXT: s_lshl_b32 s26, s17, 16
; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s28, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -3215,7 +3449,7 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; SI-NEXT: s_lshl_b32 s26, s17, 16
; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s28, s16, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
@@ -3249,33 +3483,23 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v8i32_to_v16bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -3284,7 +3508,7 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3294,17 +3518,23 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v8i32_to_v16bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
; GFX9-NEXT: s_add_i32 s21, s21, 3
@@ -3313,7 +3543,7 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3323,19 +3553,22 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_v8i32_to_v16bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s17, s17, 3
@@ -3344,15 +3577,13 @@ define inreg <16 x bfloat> @bitcast_v8i32_to_v16bf16_scalar(<8 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4151,7 +4382,7 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
@@ -4169,8 +4400,17 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -4219,20 +4459,25 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v16bf16_to_v8i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v8, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v8
@@ -4382,9 +4627,7 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v3, v9
; VI-NEXT: v_mov_b32_e32 v5, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4399,10 +4642,18 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -4558,9 +4809,7 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4580,11 +4829,16 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-TRUE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s7, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s7, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -4743,8 +4997,6 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB23_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB23_2
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4761,11 +5013,16 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-FAKE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s7, 16
; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -4941,8 +5198,6 @@ define inreg <8 x i32> @bitcast_v16bf16_to_v8i32_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v12, 16, v9
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB23_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB23_2
; GFX11-FAKE16-NEXT: .LBB23_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5516,7 +5771,7 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_4
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s23, 24
; SI-NEXT: s_lshr_b32 s57, s23, 16
@@ -5542,8 +5797,40 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB25_3
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -5576,7 +5863,7 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s73, s17, 24
; SI-NEXT: s_lshr_b32 s74, s17, 16
; SI-NEXT: s_lshr_b32 s75, s17, 8
-; SI-NEXT: .LBB25_3: ; %end
+; SI-NEXT: .LBB25_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s44
; SI-NEXT: v_mov_b32_e32 v2, s42
@@ -5610,38 +5897,12 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v30, s57
; SI-NEXT: v_mov_b32_e32 v31, s56
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB25_2
;
; VI-LABEL: bitcast_v8i32_to_v32i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_4
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s23, 24
; VI-NEXT: s_lshr_b32 s15, s23, 16
@@ -5667,8 +5928,40 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b64 s[6:7], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB25_3
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s19, s19, 3
@@ -5701,7 +5994,7 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s57, s17, 8
; VI-NEXT: s_lshr_b32 s58, s16, 16
; VI-NEXT: s_lshr_b32 s59, s16, 8
-; VI-NEXT: .LBB25_3: ; %end
+; VI-NEXT: .LBB25_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s59
; VI-NEXT: v_mov_b32_e32 v2, s58
@@ -5735,38 +6028,12 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v30, s15
; VI-NEXT: v_mov_b32_e32 v31, s14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB25_2
;
; GFX9-LABEL: bitcast_v8i32_to_v32i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s23, 24
; GFX9-NEXT: s_lshr_b32 s15, s23, 16
@@ -5792,8 +6059,40 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b64 s[6:7], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB25_3
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
@@ -5826,7 +6125,7 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s57, s17, 8
; GFX9-NEXT: s_lshr_b32 s58, s16, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 8
-; GFX9-NEXT: .LBB25_3: ; %end
+; GFX9-NEXT: .LBB25_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s59
; GFX9-NEXT: v_mov_b32_e32 v2, s58
@@ -5860,39 +6159,13 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v30, s15
; GFX9-NEXT: v_mov_b32_e32 v31, s14
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB25_2
;
; GFX11-LABEL: bitcast_v8i32_to_v32i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s46, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s19, 24
; GFX11-NEXT: s_lshr_b32 s13, s19, 16
@@ -5918,9 +6191,40 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_3
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB25_3
+; GFX11-NEXT: .LBB25_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr21
+; GFX11-NEXT: ; implicit-def: $sgpr20
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB25_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
@@ -5953,7 +6257,7 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s43, s1, 8
; GFX11-NEXT: s_lshr_b32 s44, s0, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: .LBB25_3: ; %end
+; GFX11-NEXT: .LBB25_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
; GFX11-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
@@ -5972,32 +6276,6 @@ define inreg <32 x i8> @bitcast_v8i32_to_v32i8_scalar(<8 x i32> inreg %a, i32 in
; GFX11-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
; GFX11-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB25_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6983,7 +7261,7 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s62, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s63, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -7057,8 +7335,17 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -7163,7 +7450,7 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -7173,9 +7460,6 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v32i8_to_v8i32_scalar:
; VI: ; %bb.0:
@@ -7200,7 +7484,7 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -7251,8 +7535,17 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v7, s7, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -7334,11 +7627,8 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v32i8_to_v8i32_scalar:
; GFX9: ; %bb.0:
@@ -7363,7 +7653,7 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -7414,8 +7704,17 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v7, s7, v9, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_or_b32_e32 v7, v8, v7
-; GFX9-NEXT: s_cbranch_execnz .LBB27_3
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -7490,11 +7789,8 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v8
; GFX9-NEXT: v_add_u32_sdwa v7, v7, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB27_3: ; %end
+; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB27_2
;
; GFX11-LABEL: bitcast_v32i8_to_v8i32_scalar:
; GFX11: ; %bb.0:
@@ -7516,7 +7812,7 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7552,9 +7848,17 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v5, v10, v6
; GFX11-NEXT: v_or_b32_e32 v6, v12, v9
; GFX11-NEXT: v_or_b32_e32 v7, v13, v11
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB27_3
+; GFX11-NEXT: .LBB27_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-NEXT: .LBB27_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -7628,11 +7932,8 @@ define inreg <8 x i32> @bitcast_v32i8_to_v8i32_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v5, v8, v9
; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
; GFX11-NEXT: v_or_b32_e32 v7, v12, v13
-; GFX11-NEXT: .LBB27_3: ; %end
+; GFX11-NEXT: .LBB27_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-NEXT: s_branch .LBB27_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7749,10 +8050,18 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -7762,9 +8071,7 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -7779,10 +8086,18 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -7792,9 +8107,7 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7809,10 +8122,18 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -7822,9 +8143,7 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7844,11 +8163,16 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -7858,8 +8182,6 @@ define inreg <4 x i64> @bitcast_v8f32_to_v4i64_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7989,10 +8311,18 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s20, s20, 3
@@ -8001,7 +8331,7 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8011,17 +8341,23 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v4i64_to_v8f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s20, s20, 3
@@ -8030,7 +8366,7 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8040,17 +8376,23 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v4i64_to_v8f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_3
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
@@ -8059,7 +8401,7 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB31_3: ; %end
+; GFX9-NEXT: .LBB31_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8069,19 +8411,22 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_4:
-; GFX9-NEXT: s_branch .LBB31_2
;
; GFX11-LABEL: bitcast_v4i64_to_v8f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_3
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
@@ -8090,15 +8435,13 @@ define inreg <8 x float> @bitcast_v4i64_to_v8f32_scalar(<4 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB31_3: ; %end
+; GFX11-NEXT: .LBB31_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_4:
-; GFX11-NEXT: s_branch .LBB31_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8215,10 +8558,18 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8228,9 +8579,7 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8245,10 +8594,18 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8258,9 +8615,7 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8275,10 +8630,18 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8288,9 +8651,7 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8310,11 +8671,16 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -8324,8 +8690,6 @@ define inreg <4 x double> @bitcast_v8f32_to_v4f64_scalar(<8 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8439,18 +8803,24 @@ define inreg <8 x float> @bitcast_v4f64_to_v8f32_scalar(<4 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8465,18 +8835,24 @@ define inreg <8 x float> @bitcast_v4f64_to_v8f32_scalar(<4 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8491,18 +8867,24 @@ define inreg <8 x float> @bitcast_v4f64_to_v8f32_scalar(<4 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8522,18 +8904,21 @@ define inreg <8 x float> @bitcast_v4f64_to_v8f32_scalar(<4 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8707,7 +9092,7 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s24, s21, 16
@@ -8717,8 +9102,24 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8735,18 +9136,8 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
-; SI-NEXT: s_branch .LBB37_5
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: s_branch .LBB37_6
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8763,7 +9154,7 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v9, s6
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v11, s10
-; SI-NEXT: .LBB37_5: ; %end
+; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -8794,10 +9185,18 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8807,9 +9206,7 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8824,10 +9221,18 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -8837,9 +9242,7 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8859,11 +9262,16 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -8873,8 +9281,6 @@ define inreg <16 x i16> @bitcast_v8f32_to_v16i16_scalar(<8 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9118,7 +9524,7 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -9144,8 +9550,17 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -9186,7 +9601,7 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -9196,18 +9611,23 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v16i16_to_v8f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s23, 3
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -9248,7 +9668,7 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9258,17 +9678,23 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v16i16_to_v8f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -9278,9 +9704,7 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9300,11 +9724,16 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -9314,8 +9743,6 @@ define inreg <8 x float> @bitcast_v16i16_to_v8f32_scalar(<16 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9489,7 +9916,7 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s24, s21, 16
@@ -9499,8 +9926,24 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -9517,18 +9960,8 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v1
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -9545,7 +9978,7 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v9, s6
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v11, s10
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -9576,10 +10009,18 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -9589,9 +10030,7 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9606,10 +10045,18 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -9619,9 +10066,7 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9641,11 +10086,16 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -9655,8 +10105,6 @@ define inreg <16 x half> @bitcast_v8f32_to_v16f16_scalar(<8 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9932,7 +10380,7 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_3
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -9958,8 +10406,17 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB43_4
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s40
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -10025,10 +10482,7 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB43_2
-; SI-NEXT: .LBB43_4:
+; SI-NEXT: .LBB43_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -10043,10 +10497,18 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -10089,9 +10551,7 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10106,10 +10566,18 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -10120,9 +10588,7 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10142,11 +10608,16 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -10156,8 +10627,6 @@ define inreg <8 x float> @bitcast_v16f16_to_v8f32_scalar(<16 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10372,7 +10841,7 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s23, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s23, 16
@@ -10390,8 +10859,32 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; SI-NEXT: s_lshl_b32 s27, s17, 16
; SI-NEXT: s_and_b32 s28, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s29, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
@@ -10416,26 +10909,8 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
; SI-NEXT: v_mov_b32_e32 v0, s29
; SI-NEXT: v_mov_b32_e32 v1, s28
; SI-NEXT: v_mov_b32_e32 v2, s27
@@ -10452,7 +10927,7 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v8, s7
; SI-NEXT: v_mov_b32_e32 v9, s6
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -10491,10 +10966,18 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
; VI-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -10504,9 +10987,7 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10521,10 +11002,18 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s21, 1.0
@@ -10534,9 +11023,7 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10556,11 +11043,16 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v7, s7, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s6, 1.0
; GFX11-NEXT: v_add_f32_e64 v5, s5, 1.0
@@ -10570,8 +11062,6 @@ define inreg <16 x bfloat> @bitcast_v8f32_to_v16bf16_scalar(<8 x float> inreg %a
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11376,7 +11866,7 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB47_4
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
@@ -11394,8 +11884,17 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_cbranch_execnz .LBB47_3
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -11444,20 +11943,25 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB47_3: ; %end
+; SI-NEXT: .LBB47_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SI-NEXT: s_branch .LBB47_2
;
; VI-LABEL: bitcast_v16bf16_to_v8f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v8, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v8
@@ -11607,9 +12111,7 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v3, v9
; VI-NEXT: v_mov_b32_e32 v5, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11624,10 +12126,18 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -11783,9 +12293,7 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11805,11 +12313,16 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-TRUE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s7, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s7, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -11968,8 +12481,6 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB47_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB47_2
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11986,11 +12497,16 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-FAKE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s7, 16
; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -12166,8 +12682,6 @@ define inreg <8 x float> @bitcast_v16bf16_to_v8f32_scalar(<16 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v12, 16, v9
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB47_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB47_2
; GFX11-FAKE16-NEXT: .LBB47_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12737,7 +13251,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s23, 24
; SI-NEXT: s_lshr_b32 s74, s23, 16
@@ -12763,8 +13277,40 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v39, s17, 1.0
; SI-NEXT: v_add_f32_e64 v38, s16, 1.0
; SI-NEXT: v_add_f32_e64 v49, s23, 1.0
@@ -12797,34 +13343,8 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v39
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v39
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v38, s16
; SI-NEXT: v_mov_b32_e32 v39, s17
; SI-NEXT: v_mov_b32_e32 v34, s18
@@ -12857,7 +13377,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v38
; SI-NEXT: v_mov_b32_e32 v4, v39
@@ -12876,7 +13396,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s23, 24
; VI-NEXT: s_lshr_b32 s15, s23, 16
@@ -12902,8 +13422,40 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: v_add_f32_e64 v9, s19, 1.0
@@ -12936,34 +13488,8 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v8, s18
@@ -12996,7 +13522,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v19, s8
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_mov_b32_e32 v4, v1
; VI-NEXT: v_mov_b32_e32 v12, v9
; VI-NEXT: v_mov_b32_e32 v20, v17
@@ -13011,7 +13537,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s23, 24
; GFX9-NEXT: s_lshr_b32 s15, s23, 16
@@ -13037,8 +13563,40 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s19, 1.0
@@ -13071,34 +13629,8 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -13131,7 +13663,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v19, s8
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_mov_b32_e32 v12, v9
; GFX9-NEXT: v_mov_b32_e32 v20, v17
@@ -13147,7 +13679,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s13, s19, 24
; GFX11-NEXT: s_lshr_b32 s14, s19, 16
@@ -13173,43 +13705,9 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v39, s1, 1.0
-; GFX11-NEXT: v_add_f32_e64 v37, s3, 1.0
-; GFX11-NEXT: v_add_f32_e64 v35, s17, 1.0
-; GFX11-NEXT: v_add_f32_e64 v33, s19, 1.0
-; GFX11-NEXT: v_add_f32_e64 v32, s18, 1.0
-; GFX11-NEXT: v_add_f32_e64 v34, s16, 1.0
-; GFX11-NEXT: v_add_f32_e64 v36, s2, 1.0
-; GFX11-NEXT: v_add_f32_e64 v38, s0, 1.0
-; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
-; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
-; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
-; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
-; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
-; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
+; GFX11-NEXT: s_branch .LBB49_3
+; GFX11-NEXT: .LBB49_2:
+; GFX11-NEXT: s_mov_b32 s12, -1
; GFX11-NEXT: ; implicit-def: $sgpr46
; GFX11-NEXT: ; implicit-def: $sgpr44
; GFX11-NEXT: ; implicit-def: $sgpr4
@@ -13234,8 +13732,47 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: ; implicit-def: $sgpr20
; GFX11-NEXT: ; implicit-def: $sgpr14
; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: s_branch .LBB49_2
-; GFX11-NEXT: .LBB49_4:
+; GFX11-NEXT: .LBB49_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_add_f32_e64 v39, s1, 1.0
+; GFX11-NEXT: v_add_f32_e64 v37, s3, 1.0
+; GFX11-NEXT: v_add_f32_e64 v35, s17, 1.0
+; GFX11-NEXT: v_add_f32_e64 v33, s19, 1.0
+; GFX11-NEXT: v_add_f32_e64 v32, s18, 1.0
+; GFX11-NEXT: v_add_f32_e64 v34, s16, 1.0
+; GFX11-NEXT: v_add_f32_e64 v36, s2, 1.0
+; GFX11-NEXT: v_add_f32_e64 v38, s0, 1.0
+; GFX11-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-NEXT: s_branch .LBB49_6
+; GFX11-NEXT: .LBB49_5:
; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
@@ -13256,7 +13793,7 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v19, s8
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB49_5: ; %end
+; GFX11-NEXT: .LBB49_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v38
; GFX11-NEXT: v_mov_b32_e32 v4, v39
; GFX11-NEXT: v_mov_b32_e32 v8, v36
@@ -14251,7 +14788,7 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s62, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s63, v0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -14325,8 +14862,17 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -14431,7 +14977,7 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -14441,9 +14987,6 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v32i8_to_v8f32_scalar:
; VI: ; %bb.0:
@@ -14468,7 +15011,7 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -14519,8 +15062,17 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v7, s7, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -14602,11 +15154,8 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v32i8_to_v8f32_scalar:
; GFX9: ; %bb.0:
@@ -14631,7 +15180,7 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -14682,8 +15231,17 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v7, s7, v9, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_or_b32_e32 v7, v8, v7
-; GFX9-NEXT: s_cbranch_execnz .LBB51_3
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -14758,11 +15316,8 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v8
; GFX9-NEXT: v_add_u32_sdwa v7, v7, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB51_3: ; %end
+; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB51_2
;
; GFX11-LABEL: bitcast_v32i8_to_v8f32_scalar:
; GFX11: ; %bb.0:
@@ -14784,7 +15339,7 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14820,9 +15375,17 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v5, v10, v6
; GFX11-NEXT: v_or_b32_e32 v6, v12, v9
; GFX11-NEXT: v_or_b32_e32 v7, v13, v11
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB51_3
+; GFX11-NEXT: .LBB51_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-NEXT: .LBB51_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -14896,11 +15459,8 @@ define inreg <8 x float> @bitcast_v32i8_to_v8f32_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v5, v8, v9
; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
; GFX11-NEXT: v_or_b32_e32 v7, v12, v13
-; GFX11-NEXT: .LBB51_3: ; %end
+; GFX11-NEXT: .LBB51_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-NEXT: s_branch .LBB51_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -15024,10 +15584,18 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -15036,7 +15604,7 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s21, s21, 0
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15046,17 +15614,23 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s22
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v4i64_to_v4f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -15065,7 +15639,7 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s21, s21, 0
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15075,17 +15649,23 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v4i64_to_v4f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -15094,7 +15674,7 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s21, s21, 0
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15104,19 +15684,22 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v4i64_to_v4f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -15125,14 +15708,12 @@ define inreg <4 x double> @bitcast_v4i64_to_v4f64_scalar(<4 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s17, s17, 0
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -15240,18 +15821,24 @@ define inreg <4 x i64> @bitcast_v4f64_to_v4i64_scalar(<4 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15266,18 +15853,24 @@ define inreg <4 x i64> @bitcast_v4f64_to_v4i64_scalar(<4 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15292,18 +15885,24 @@ define inreg <4 x i64> @bitcast_v4f64_to_v4i64_scalar(<4 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15323,18 +15922,21 @@ define inreg <4 x i64> @bitcast_v4f64_to_v4i64_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15515,7 +16117,7 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s23, 16
; SI-NEXT: s_lshr_b32 s15, s21, 16
@@ -15525,8 +16127,24 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s20, s20, 3
@@ -15543,7 +16161,7 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -15577,25 +16195,23 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v4i64_to_v16i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s20, s20, 3
@@ -15604,7 +16220,7 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15614,17 +16230,23 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v4i64_to_v16i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_3
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
@@ -15633,7 +16255,7 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB57_3: ; %end
+; GFX9-NEXT: .LBB57_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15643,19 +16265,22 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: s_branch .LBB57_2
;
; GFX11-LABEL: bitcast_v4i64_to_v16i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_3
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
@@ -15664,15 +16289,13 @@ define inreg <16 x i16> @bitcast_v4i64_to_v16i16_scalar(<4 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB57_3: ; %end
+; GFX11-NEXT: .LBB57_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_4:
-; GFX11-NEXT: s_branch .LBB57_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -15910,7 +16533,7 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -15936,8 +16559,17 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -15978,7 +16610,7 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -15988,18 +16620,23 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v16i16_to_v4i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s23, 3
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -16040,7 +16677,7 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16050,17 +16687,23 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v16i16_to_v4i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -16070,9 +16713,7 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16092,11 +16733,16 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -16106,8 +16752,6 @@ define inreg <4 x i64> @bitcast_v16i16_to_v4i64_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -16288,7 +16932,7 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB61_4
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s23, 16
; SI-NEXT: s_lshr_b32 s15, s21, 16
@@ -16298,8 +16942,24 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB61_3
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s20, s20, 3
@@ -16316,7 +16976,7 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: .LBB61_3: ; %end
+; SI-NEXT: .LBB61_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s10, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -16350,25 +17010,23 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB61_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB61_2
;
; VI-LABEL: bitcast_v4i64_to_v16f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB61_4
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_3
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s20, s20, 3
@@ -16377,7 +17035,7 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB61_3: ; %end
+; VI-NEXT: .LBB61_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16387,17 +17045,23 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_4:
-; VI-NEXT: s_branch .LBB61_2
;
; GFX9-LABEL: bitcast_v4i64_to_v16f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_3
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
@@ -16406,7 +17070,7 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB61_3: ; %end
+; GFX9-NEXT: .LBB61_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16416,19 +17080,22 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_4:
-; GFX9-NEXT: s_branch .LBB61_2
;
; GFX11-LABEL: bitcast_v4i64_to_v16f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_3
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
@@ -16437,15 +17104,13 @@ define inreg <16 x half> @bitcast_v4i64_to_v16f16_scalar(<4 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB61_3: ; %end
+; GFX11-NEXT: .LBB61_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_4:
-; GFX11-NEXT: s_branch .LBB61_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16715,7 +17380,7 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB63_3
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -16741,8 +17406,17 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB63_4
-; SI-NEXT: .LBB63_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s40
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -16808,10 +17482,7 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB63_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB63_2
-; SI-NEXT: .LBB63_4:
+; SI-NEXT: .LBB63_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -16826,10 +17497,18 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -16872,9 +17551,7 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16889,10 +17566,18 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -16903,9 +17588,7 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16925,11 +17608,16 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB63_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -16939,8 +17627,6 @@ define inreg <4 x i64> @bitcast_v16f16_to_v4i64_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB63_3:
-; GFX11-NEXT: s_branch .LBB63_2
; GFX11-NEXT: .LBB63_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -17162,7 +17848,7 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB65_4
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s23, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s23, 16
@@ -17180,8 +17866,32 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; SI-NEXT: s_lshl_b32 s26, s17, 16
; SI-NEXT: s_and_b32 s29, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s28, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB65_3
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s16, 3
; SI-NEXT: s_addc_u32 s5, s17, 0
; SI-NEXT: s_add_u32 s16, s18, 3
@@ -17206,7 +17916,7 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; SI-NEXT: s_lshl_b32 s26, s5, 16
; SI-NEXT: s_and_b32 s29, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s28, s4, 16
-; SI-NEXT: .LBB65_3: ; %end
+; SI-NEXT: .LBB65_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s28
@@ -17240,33 +17950,23 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_4:
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB65_2
;
; VI-LABEL: bitcast_v4i64_to_v16bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB65_4
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_3
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s22, s22, 3
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s20, s20, 3
@@ -17275,7 +17975,7 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB65_3: ; %end
+; VI-NEXT: .LBB65_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17285,17 +17985,23 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_4:
-; VI-NEXT: s_branch .LBB65_2
;
; GFX9-LABEL: bitcast_v4i64_to_v16bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_3
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s22, s22, 3
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s20, s20, 3
@@ -17304,7 +18010,7 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB65_3: ; %end
+; GFX9-NEXT: .LBB65_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17314,19 +18020,22 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v6, s22
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_4:
-; GFX9-NEXT: s_branch .LBB65_2
;
; GFX11-LABEL: bitcast_v4i64_to_v16bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_3
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s18, s18, 3
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s16, s16, 3
@@ -17335,15 +18044,13 @@ define inreg <16 x bfloat> @bitcast_v4i64_to_v16bf16_scalar(<4 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB65_3: ; %end
+; GFX11-NEXT: .LBB65_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_4:
-; GFX11-NEXT: s_branch .LBB65_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -18142,7 +18849,7 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB67_4
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
@@ -18160,8 +18867,17 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_cbranch_execnz .LBB67_3
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -18210,20 +18926,25 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB67_3: ; %end
+; SI-NEXT: .LBB67_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB67_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SI-NEXT: s_branch .LBB67_2
;
; VI-LABEL: bitcast_v16bf16_to_v4i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v8, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v8
@@ -18373,9 +19094,7 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v3, v9
; VI-NEXT: v_mov_b32_e32 v5, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -18390,10 +19109,18 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -18549,9 +19276,7 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -18571,11 +19296,16 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-TRUE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s7, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s7, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -18734,8 +19464,6 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB67_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB67_2
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -18752,11 +19480,16 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-FAKE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s7, 16
; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -18932,8 +19665,6 @@ define inreg <4 x i64> @bitcast_v16bf16_to_v4i64_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v12, 16, v9
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB67_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB67_2
; GFX11-FAKE16-NEXT: .LBB67_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -19513,7 +20244,7 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB69_4
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s23, 24
; SI-NEXT: s_lshr_b32 s57, s23, 16
@@ -19539,8 +20270,40 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB69_3
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s22, s22, 3
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s20, s20, 3
@@ -19573,7 +20336,7 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 8
-; SI-NEXT: .LBB69_3: ; %end
+; SI-NEXT: .LBB69_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s44
; SI-NEXT: v_mov_b32_e32 v2, s42
@@ -19607,38 +20370,12 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v30, s57
; SI-NEXT: v_mov_b32_e32 v31, s56
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB69_2
;
; VI-LABEL: bitcast_v4i64_to_v32i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB69_4
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s23, 24
; VI-NEXT: s_lshr_b32 s15, s23, 16
@@ -19664,8 +20401,40 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b64 s[6:7], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB69_3
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -19698,7 +20467,7 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s57, s17, 8
; VI-NEXT: s_lshr_b32 s58, s16, 16
; VI-NEXT: s_lshr_b32 s59, s16, 8
-; VI-NEXT: .LBB69_3: ; %end
+; VI-NEXT: .LBB69_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s59
; VI-NEXT: v_mov_b32_e32 v2, s58
@@ -19732,38 +20501,12 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v30, s15
; VI-NEXT: v_mov_b32_e32 v31, s14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB69_2
;
; GFX9-LABEL: bitcast_v4i64_to_v32i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s23, 24
; GFX9-NEXT: s_lshr_b32 s15, s23, 16
@@ -19789,8 +20532,40 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b64 s[6:7], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB69_3
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -19823,7 +20598,7 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s57, s17, 8
; GFX9-NEXT: s_lshr_b32 s58, s16, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 8
-; GFX9-NEXT: .LBB69_3: ; %end
+; GFX9-NEXT: .LBB69_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s59
; GFX9-NEXT: v_mov_b32_e32 v2, s58
@@ -19857,39 +20632,13 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v30, s15
; GFX9-NEXT: v_mov_b32_e32 v31, s14
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB69_2
;
; GFX11-LABEL: bitcast_v4i64_to_v32i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s46, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s19, 24
; GFX11-NEXT: s_lshr_b32 s13, s19, 16
@@ -19915,9 +20664,40 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[8:9], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[10:11], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_3
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr21
+; GFX11-NEXT: ; implicit-def: $sgpr20
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -19950,7 +20730,7 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s43, s1, 8
; GFX11-NEXT: s_lshr_b32 s44, s0, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 8
-; GFX11-NEXT: .LBB69_3: ; %end
+; GFX11-NEXT: .LBB69_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s45
; GFX11-NEXT: v_dual_mov_b32 v2, s44 :: v_dual_mov_b32 v3, s10
@@ -19969,32 +20749,6 @@ define inreg <32 x i8> @bitcast_v4i64_to_v32i8_scalar(<4 x i64> inreg %a, i32 in
; GFX11-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s14
; GFX11-NEXT: v_dual_mov_b32 v30, s13 :: v_dual_mov_b32 v31, s12
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB69_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -20980,7 +21734,7 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s62, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s63, v0
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -21054,8 +21808,17 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -21160,7 +21923,7 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -21170,9 +21933,6 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v32i8_to_v4i64_scalar:
; VI: ; %bb.0:
@@ -21197,7 +21957,7 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -21248,8 +22008,17 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v7, s7, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -21331,11 +22100,8 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v32i8_to_v4i64_scalar:
; GFX9: ; %bb.0:
@@ -21360,7 +22126,7 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -21411,8 +22177,17 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v7, s7, v9, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_or_b32_e32 v7, v8, v7
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -21487,11 +22262,8 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v8
; GFX9-NEXT: v_add_u32_sdwa v7, v7, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-LABEL: bitcast_v32i8_to_v4i64_scalar:
; GFX11: ; %bb.0:
@@ -21513,7 +22285,7 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -21549,9 +22321,17 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v5, v10, v6
; GFX11-NEXT: v_or_b32_e32 v6, v12, v9
; GFX11-NEXT: v_or_b32_e32 v7, v13, v11
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -21625,11 +22405,8 @@ define inreg <4 x i64> @bitcast_v32i8_to_v4i64_scalar(<32 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v5, v8, v9
; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
; GFX11-NEXT: v_or_b32_e32 v7, v12, v13
-; GFX11-NEXT: .LBB71_3: ; %end
+; GFX11-NEXT: .LBB71_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB71_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -21788,7 +22565,7 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB73_3
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s24, s21, 16
@@ -21798,8 +22575,24 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB73_4
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
@@ -21812,18 +22605,8 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: s_branch .LBB73_5
-; SI-NEXT: .LBB73_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: s_branch .LBB73_2
-; SI-NEXT: .LBB73_4:
+; SI-NEXT: s_branch .LBB73_6
+; SI-NEXT: .LBB73_5:
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -21840,7 +22623,7 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v9, s6
; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -21871,18 +22654,24 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
+; VI-NEXT: .LBB73_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -21897,18 +22686,24 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -21928,18 +22723,21 @@ define inreg <16 x i16> @bitcast_v4f64_to_v16i16_scalar(<4 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB73_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB73_3:
-; GFX11-NEXT: s_branch .LBB73_2
; GFX11-NEXT: .LBB73_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22183,7 +22981,7 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_4
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -22209,8 +23007,17 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB75_3
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -22251,7 +23058,7 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s9, s9, 0x30000
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: .LBB75_3: ; %end
+; SI-NEXT: .LBB75_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -22261,18 +23068,23 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB75_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB75_2
;
; VI-LABEL: bitcast_v16i16_to_v4f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB75_4
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB75_3
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s23, 3
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -22313,7 +23125,7 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB75_3: ; %end
+; VI-NEXT: .LBB75_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -22323,17 +23135,23 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB75_4:
-; VI-NEXT: s_branch .LBB75_2
;
; GFX9-LABEL: bitcast_v16i16_to_v4f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB75_4
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -22343,9 +23161,7 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB75_3:
-; GFX9-NEXT: s_branch .LBB75_2
-; GFX9-NEXT: .LBB75_4:
+; GFX9-NEXT: .LBB75_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -22365,11 +23181,16 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB75_4
-; GFX11-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB75_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -22379,8 +23200,6 @@ define inreg <4 x double> @bitcast_v16i16_to_v4f64_scalar(<16 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: s_branch .LBB75_2
; GFX11-NEXT: .LBB75_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22545,7 +23364,7 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB77_3
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s24, s21, 16
@@ -22555,8 +23374,24 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB77_4
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
@@ -22569,18 +23404,8 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: s_branch .LBB77_5
-; SI-NEXT: .LBB77_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: s_branch .LBB77_2
-; SI-NEXT: .LBB77_4:
+; SI-NEXT: s_branch .LBB77_6
+; SI-NEXT: .LBB77_5:
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: v_mov_b32_e32 v5, s21
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -22597,7 +23422,7 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: v_mov_b32_e32 v9, s6
; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: .LBB77_5: ; %end
+; SI-NEXT: .LBB77_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -22628,18 +23453,24 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -22654,18 +23485,24 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -22685,18 +23522,21 @@ define inreg <16 x half> @bitcast_v4f64_to_v16f16_scalar(<4 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB77_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB77_3:
-; GFX11-NEXT: s_branch .LBB77_2
; GFX11-NEXT: .LBB77_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22972,7 +23812,7 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_3
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s40, 16
@@ -22998,8 +23838,17 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; SI-NEXT: s_and_b32 s11, s23, 0xffff
; SI-NEXT: s_lshl_b32 s12, s14, 16
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB79_4
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s40
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -23065,10 +23914,7 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB79_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB79_2
-; SI-NEXT: .LBB79_4:
+; SI-NEXT: .LBB79_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -23083,10 +23929,18 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB79_3
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB79_4
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -23129,9 +23983,7 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB79_3:
-; VI-NEXT: s_branch .LBB79_2
-; VI-NEXT: .LBB79_4:
+; VI-NEXT: .LBB79_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -23146,10 +23998,18 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB79_4
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -23160,9 +24020,7 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB79_3:
-; GFX9-NEXT: s_branch .LBB79_2
-; GFX9-NEXT: .LBB79_4:
+; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -23182,11 +24040,16 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB79_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -23196,8 +24059,6 @@ define inreg <4 x double> @bitcast_v16f16_to_v4f64_scalar(<16 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_3:
-; GFX11-NEXT: s_branch .LBB79_2
; GFX11-NEXT: .LBB79_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -23399,7 +24260,7 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s29, s23, 0xffff0000
; SI-NEXT: s_lshl_b32 s28, s23, 16
@@ -23417,8 +24278,32 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; SI-NEXT: s_lshl_b32 s8, s17, 16
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[22:23], 1.0
@@ -23439,26 +24324,8 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB81_5
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: s_branch .LBB81_6
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v9, s29
; SI-NEXT: v_mov_b32_e32 v8, s28
; SI-NEXT: v_mov_b32_e32 v10, s27
@@ -23475,7 +24342,7 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: .LBB81_5: ; %end
+; SI-NEXT: .LBB81_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -23514,18 +24381,24 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -23540,18 +24413,24 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -23571,18 +24450,21 @@ define inreg <16 x bfloat> @bitcast_v4f64_to_v16bf16_scalar(<4 x double> inreg %
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[6:7], s[6:7], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[4:5], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -24387,7 +25269,7 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v31
@@ -24405,8 +25287,17 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_lshr_b64 v[6:7], v[11:12], 16
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v25
; SI-NEXT: v_lshr_b64 v[7:8], v[9:10], 16
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -24455,20 +25346,25 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v16bf16_to_v4f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB83_3
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_4
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_mov_b32_e32 v8, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v8
@@ -24618,9 +25514,7 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_mov_b32_e32 v3, v9
; VI-NEXT: v_mov_b32_e32 v5, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB83_3:
-; VI-NEXT: s_branch .LBB83_2
-; VI-NEXT: .LBB83_4:
+; VI-NEXT: .LBB83_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -24635,10 +25529,18 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_4
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s23, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -24794,9 +25696,7 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_and_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v9, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB83_3:
-; GFX9-NEXT: s_branch .LBB83_2
-; GFX9-NEXT: .LBB83_4:
+; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -24816,11 +25716,16 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-TRUE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s7, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s7, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -24979,8 +25884,6 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB83_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB83_2
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -24997,11 +25900,16 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-FAKE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s8, s7, 16
; GFX11-FAKE16-NEXT: s_and_b32 s7, s7, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -25177,8 +26085,6 @@ define inreg <4 x double> @bitcast_v16bf16_to_v4f64_scalar(<16 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v12, 16, v9
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB83_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB83_2
; GFX11-FAKE16-NEXT: .LBB83_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -25742,7 +26648,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB85_3
+; SI-NEXT: s_cbranch_scc0 .LBB85_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s75, s23, 24
; SI-NEXT: s_lshr_b32 s74, s23, 16
@@ -25768,8 +26674,40 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB85_4
-; SI-NEXT: .LBB85_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB85_3
+; SI-NEXT: .LBB85_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: .LBB85_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB85_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[50:51], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[37:38], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[48:49], s[16:17], 1.0
@@ -25798,34 +26736,8 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v49
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v49
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v49
-; SI-NEXT: s_branch .LBB85_5
-; SI-NEXT: .LBB85_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: s_branch .LBB85_2
-; SI-NEXT: .LBB85_4:
+; SI-NEXT: s_branch .LBB85_6
+; SI-NEXT: .LBB85_5:
; SI-NEXT: v_mov_b32_e32 v51, s23
; SI-NEXT: v_mov_b32_e32 v38, s21
; SI-NEXT: v_mov_b32_e32 v36, s19
@@ -25858,7 +26770,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v25, s8
; SI-NEXT: v_mov_b32_e32 v24, s6
; SI-NEXT: v_mov_b32_e32 v27, s4
-; SI-NEXT: .LBB85_5: ; %end
+; SI-NEXT: .LBB85_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v48
; SI-NEXT: v_mov_b32_e32 v4, v49
@@ -25877,7 +26789,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB85_3
+; VI-NEXT: s_cbranch_scc0 .LBB85_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s43, s23, 24
; VI-NEXT: s_lshr_b32 s42, s23, 16
@@ -25903,8 +26815,40 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB85_4
-; VI-NEXT: .LBB85_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB85_3
+; VI-NEXT: .LBB85_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: .LBB85_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB85_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
@@ -25933,34 +26877,8 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; VI-NEXT: s_branch .LBB85_5
-; VI-NEXT: .LBB85_3:
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: s_branch .LBB85_2
-; VI-NEXT: .LBB85_4:
+; VI-NEXT: s_branch .LBB85_6
+; VI-NEXT: .LBB85_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v8, s18
; VI-NEXT: v_mov_b32_e32 v16, s20
@@ -25993,7 +26911,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v19, s8
; VI-NEXT: v_mov_b32_e32 v27, s10
-; VI-NEXT: .LBB85_5: ; %end
+; VI-NEXT: .LBB85_6: ; %end
; VI-NEXT: v_mov_b32_e32 v4, v1
; VI-NEXT: v_mov_b32_e32 v12, v9
; VI-NEXT: v_mov_b32_e32 v20, v17
@@ -26008,7 +26926,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB85_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s43, s23, 24
; GFX9-NEXT: s_lshr_b32 s42, s23, 16
@@ -26034,8 +26952,40 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB85_4
-; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB85_3
+; GFX9-NEXT: .LBB85_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: .LBB85_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[20:21], 1.0
@@ -26064,34 +27014,8 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX9-NEXT: s_branch .LBB85_5
-; GFX9-NEXT: .LBB85_3:
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: s_branch .LBB85_2
-; GFX9-NEXT: .LBB85_4:
+; GFX9-NEXT: s_branch .LBB85_6
+; GFX9-NEXT: .LBB85_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v8, s18
; GFX9-NEXT: v_mov_b32_e32 v16, s20
@@ -26124,7 +27048,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v19, s8
; GFX9-NEXT: v_mov_b32_e32 v27, s10
-; GFX9-NEXT: .LBB85_5: ; %end
+; GFX9-NEXT: .LBB85_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_mov_b32_e32 v12, v9
; GFX9-NEXT: v_mov_b32_e32 v20, v17
@@ -26140,7 +27064,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s28, s19, 24
; GFX11-NEXT: s_lshr_b32 s27, s19, 16
@@ -26166,9 +27090,40 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
-; GFX11-NEXT: .LBB85_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB85_3
+; GFX11-NEXT: .LBB85_2:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr20
+; GFX11-NEXT: ; implicit-def: $sgpr21
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: .LBB85_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[32:33], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[34:35], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[36:37], s[2:3], 1.0
@@ -26199,34 +27154,8 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB85_5
-; GFX11-NEXT: .LBB85_3:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: s_branch .LBB85_2
-; GFX11-NEXT: .LBB85_4:
+; GFX11-NEXT: s_branch .LBB85_6
+; GFX11-NEXT: .LBB85_5:
; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v33, s19
; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v35, s17
; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v37, s3
@@ -26247,7 +27176,7 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v13, s20
; GFX11-NEXT: v_mov_b32_e32 v7, s15
; GFX11-NEXT: v_mov_b32_e32 v5, s13
-; GFX11-NEXT: .LBB85_5: ; %end
+; GFX11-NEXT: .LBB85_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v38
; GFX11-NEXT: v_mov_b32_e32 v4, v39
; GFX11-NEXT: v_mov_b32_e32 v8, v36
@@ -27242,7 +28171,7 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s62, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s63, v0
-; SI-NEXT: s_cbranch_scc0 .LBB87_4
+; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -27316,8 +28245,17 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_cbranch_execnz .LBB87_3
-; SI-NEXT: .LBB87_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB87_3
+; SI-NEXT: .LBB87_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+; SI-NEXT: .LBB87_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB87_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -27422,7 +28360,7 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: .LBB87_3: ; %end
+; SI-NEXT: .LBB87_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -27432,9 +28370,6 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s10
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB87_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
-; SI-NEXT: s_branch .LBB87_2
;
; VI-LABEL: bitcast_v32i8_to_v4f64_scalar:
; VI: ; %bb.0:
@@ -27459,7 +28394,7 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s46, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s47, v0
-; VI-NEXT: s_cbranch_scc0 .LBB87_4
+; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -27510,8 +28445,17 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v7, s7, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB87_3
-; VI-NEXT: .LBB87_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB87_3
+; VI-NEXT: .LBB87_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB87_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB87_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -27593,11 +28537,8 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB87_3: ; %end
+; VI-NEXT: .LBB87_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB87_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB87_2
;
; GFX9-LABEL: bitcast_v32i8_to_v4f64_scalar:
; GFX9: ; %bb.0:
@@ -27622,7 +28563,7 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s46, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s47, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -27673,8 +28614,17 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v7, s7, v9, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_or_b32_e32 v7, v8, v7
-; GFX9-NEXT: s_cbranch_execnz .LBB87_3
-; GFX9-NEXT: .LBB87_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB87_3
+; GFX9-NEXT: .LBB87_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB87_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v7, 0xc0c0004
@@ -27749,11 +28699,8 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v8, 0x300, v8
; GFX9-NEXT: v_add_u32_sdwa v7, v7, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB87_3: ; %end
+; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB87_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB87_2
;
; GFX11-LABEL: bitcast_v32i8_to_v4f64_scalar:
; GFX11: ; %bb.0:
@@ -27775,7 +28722,7 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s41, v0
; GFX11-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -27811,9 +28758,17 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v5, v10, v6
; GFX11-NEXT: v_or_b32_e32 v6, v12, v9
; GFX11-NEXT: v_or_b32_e32 v7, v13, v11
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB87_3
-; GFX11-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB87_3
+; GFX11-NEXT: .LBB87_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-NEXT: .LBB87_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
; GFX11-NEXT: s_add_i32 s18, s18, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -27887,11 +28842,8 @@ define inreg <4 x double> @bitcast_v32i8_to_v4f64_scalar(<32 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v5, v8, v9
; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
; GFX11-NEXT: v_or_b32_e32 v7, v12, v13
-; GFX11-NEXT: .LBB87_3: ; %end
+; GFX11-NEXT: .LBB87_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB87_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-NEXT: s_branch .LBB87_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -28164,7 +29116,7 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s40, s17, 16
; SI-NEXT: s_lshr_b32 s44, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB89_4
+; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s40, 16
@@ -28194,12 +29146,28 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[14:15], s[58:59], 16
; SI-NEXT: s_lshr_b64 s[24:25], s[60:61], 16
; SI-NEXT: s_lshr_b64 s[26:27], s[62:63], 16
+; SI-NEXT: s_mov_b64 s[28:29], 0
; SI-NEXT: s_mov_b32 s11, s57
; SI-NEXT: s_mov_b32 s9, s59
; SI-NEXT: s_mov_b32 s7, s61
; SI-NEXT: s_mov_b32 s5, s63
-; SI-NEXT: s_cbranch_execnz .LBB89_3
-; SI-NEXT: .LBB89_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB89_3
+; SI-NEXT: .LBB89_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: .LBB89_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB89_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_and_b32 s4, s22, 0xffff
; SI-NEXT: s_lshl_b32 s5, s47, 16
@@ -28248,7 +29216,7 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s41, s9, 16
; SI-NEXT: s_lshr_b32 s42, s7, 16
; SI-NEXT: s_lshr_b32 s43, s5, 16
-; SI-NEXT: .LBB89_3: ; %end
+; SI-NEXT: .LBB89_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -28282,25 +29250,23 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v7, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB89_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: s_branch .LBB89_2
;
; VI-LABEL: bitcast_v16i16_to_v16f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB89_4
+; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB89_3
-; VI-NEXT: .LBB89_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB89_3
+; VI-NEXT: .LBB89_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB89_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB89_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_add_i32 s7, s17, 3
@@ -28341,7 +29307,7 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB89_3: ; %end
+; VI-NEXT: .LBB89_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -28351,17 +29317,23 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB89_4:
-; VI-NEXT: s_branch .LBB89_2
;
; GFX9-LABEL: bitcast_v16i16_to_v16f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB89_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB89_4
-; GFX9-NEXT: .LBB89_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB89_3
+; GFX9-NEXT: .LBB89_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB89_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -28371,9 +29343,7 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB89_3:
-; GFX9-NEXT: s_branch .LBB89_2
-; GFX9-NEXT: .LBB89_4:
+; GFX9-NEXT: .LBB89_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -28393,11 +29363,16 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB89_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB89_4
-; GFX11-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB89_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -28407,8 +29382,6 @@ define inreg <16 x half> @bitcast_v16i16_to_v16f16_scalar(<16 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB89_3:
-; GFX11-NEXT: s_branch .LBB89_2
; GFX11-NEXT: .LBB89_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -28656,10 +29629,18 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB91_3
+; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB91_4
-; SI-NEXT: .LBB91_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB91_3
+; SI-NEXT: .LBB91_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB91_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB91_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s13
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -28728,10 +29709,8 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[14:15], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[6:7], 16
-; SI-NEXT: s_branch .LBB91_5
-; SI-NEXT: .LBB91_3:
-; SI-NEXT: s_branch .LBB91_2
-; SI-NEXT: .LBB91_4:
+; SI-NEXT: s_branch .LBB91_6
+; SI-NEXT: .LBB91_5:
; SI-NEXT: v_mov_b32_e32 v17, s10
; SI-NEXT: v_mov_b32_e32 v21, s8
; SI-NEXT: v_mov_b32_e32 v22, s7
@@ -28748,7 +29727,7 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s12
; SI-NEXT: v_mov_b32_e32 v10, s11
; SI-NEXT: v_mov_b32_e32 v8, s9
-; SI-NEXT: .LBB91_5: ; %end
+; SI-NEXT: .LBB91_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -28779,10 +29758,18 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB91_3
+; VI-NEXT: s_cbranch_scc0 .LBB91_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB91_4
-; VI-NEXT: .LBB91_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB91_3
+; VI-NEXT: .LBB91_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB91_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB91_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -28825,9 +29812,7 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v10
; VI-NEXT: v_or_b32_e32 v0, v8, v9
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB91_3:
-; VI-NEXT: s_branch .LBB91_2
-; VI-NEXT: .LBB91_4:
+; VI-NEXT: .LBB91_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -28842,10 +29827,18 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB91_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB91_4
-; GFX9-NEXT: .LBB91_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB91_3
+; GFX9-NEXT: .LBB91_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB91_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -28856,9 +29849,7 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB91_3:
-; GFX9-NEXT: s_branch .LBB91_2
-; GFX9-NEXT: .LBB91_4:
+; GFX9-NEXT: .LBB91_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -28878,11 +29869,16 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB91_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB91_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -28892,8 +29888,6 @@ define inreg <16 x i16> @bitcast_v16f16_to_v16i16_scalar(<16 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB91_3:
-; GFX11-NEXT: s_branch .LBB91_2
; GFX11-NEXT: .LBB91_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -29163,7 +30157,7 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-NEXT: s_lshr_b32 s41, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB93_4
+; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s25, s16, 16
; SI-NEXT: s_lshl_b32 s29, s40, 16
@@ -29181,8 +30175,32 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-NEXT: s_lshl_b32 s12, s46, 16
; SI-NEXT: s_lshl_b32 s6, s23, 16
; SI-NEXT: s_lshl_b32 s9, s47, 16
-; SI-NEXT: s_cbranch_execnz .LBB93_3
-; SI-NEXT: .LBB93_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB93_3
+; SI-NEXT: .LBB93_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB93_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB93_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_and_b32 s4, s23, 0xffff
; SI-NEXT: s_lshl_b32 s5, s47, 16
@@ -29239,7 +30257,7 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s4, 16
-; SI-NEXT: .LBB93_3: ; %end
+; SI-NEXT: .LBB93_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s29
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s25
@@ -29273,33 +30291,23 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB93_4:
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB93_2
;
; VI-LABEL: bitcast_v16i16_to_v16bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB93_4
+; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB93_3
-; VI-NEXT: .LBB93_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB93_3
+; VI-NEXT: .LBB93_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB93_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB93_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_add_i32 s7, s17, 3
@@ -29340,7 +30348,7 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB93_3: ; %end
+; VI-NEXT: .LBB93_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -29350,17 +30358,23 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; VI-NEXT: v_mov_b32_e32 v6, s22
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB93_4:
-; VI-NEXT: s_branch .LBB93_2
;
; GFX9-LABEL: bitcast_v16i16_to_v16bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB93_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB93_4
-; GFX9-NEXT: .LBB93_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB93_3
+; GFX9-NEXT: .LBB93_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB93_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s21, 3 op_sel_hi:[1,0]
@@ -29370,9 +30384,7 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB93_3:
-; GFX9-NEXT: s_branch .LBB93_2
-; GFX9-NEXT: .LBB93_4:
+; GFX9-NEXT: .LBB93_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -29392,11 +30404,16 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB93_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB93_4
-; GFX11-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB93_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v7, s7, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s6, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v5, s5, 3 op_sel_hi:[1,0]
@@ -29406,8 +30423,6 @@ define inreg <16 x bfloat> @bitcast_v16i16_to_v16bf16_scalar(<16 x i16> inreg %a
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB93_3:
-; GFX11-NEXT: s_branch .LBB93_2
; GFX11-NEXT: .LBB93_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -30235,11 +31250,11 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: v_mul_f32_e64 v31, 1.0, s16
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v30, 1.0, s15
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s13
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v29, 1.0, s11
; SI-NEXT: v_mul_f32_e64 v28, 1.0, s10
@@ -30247,33 +31262,57 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s8
; SI-NEXT: v_mul_f32_e64 v27, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB95_4
+; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v31
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v2
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v29
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v28
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v10
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v19
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v19
-; SI-NEXT: s_cbranch_execnz .LBB95_3
-; SI-NEXT: .LBB95_2: ; %cmp.true
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v21
+; SI-NEXT: s_branch .LBB95_3
+; SI-NEXT: .LBB95_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr18
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: .LBB95_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB95_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
@@ -30281,11 +31320,11 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v28
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v7
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v11
; SI-NEXT: v_lshr_b64 v[13:14], v[6:7], 16
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v27
@@ -30293,39 +31332,39 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v9
; SI-NEXT: v_lshr_b64 v[14:15], v[6:7], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v19
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v6
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v8
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v1
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v9
; SI-NEXT: v_lshr_b64 v[9:10], v[17:18], 16
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v1
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v10
; SI-NEXT: v_lshr_b64 v[11:12], v[21:22], 16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v23
; SI-NEXT: v_lshr_b64 v[23:24], v[10:11], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[8:9], 16
; SI-NEXT: v_lshr_b64 v[26:27], v[6:7], 16
-; SI-NEXT: .LBB95_3: ; %end
+; SI-NEXT: .LBB95_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v23
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v11
@@ -30351,33 +31390,23 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB95_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: s_branch .LBB95_2
;
; VI-LABEL: bitcast_v16bf16_to_v16i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB95_3
+; VI-NEXT: s_cbranch_scc0 .LBB95_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB95_4
-; VI-NEXT: .LBB95_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB95_3
+; VI-NEXT: .LBB95_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB95_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB95_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v10, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v10
@@ -30528,9 +31557,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v5, v13
; VI-NEXT: v_mov_b32_e32 v7, v15
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB95_3:
-; VI-NEXT: s_branch .LBB95_2
-; VI-NEXT: .LBB95_4:
+; VI-NEXT: .LBB95_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -30545,10 +31572,18 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB95_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB95_4
-; GFX9-NEXT: .LBB95_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB95_3
+; GFX9-NEXT: .LBB95_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB95_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -30696,9 +31731,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX9-NEXT: v_and_or_b32 v1, v9, v13, v1
; GFX9-NEXT: v_and_or_b32 v0, v0, v13, v8
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB95_3:
-; GFX9-NEXT: s_branch .LBB95_2
-; GFX9-NEXT: .LBB95_4:
+; GFX9-NEXT: .LBB95_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -30718,11 +31751,16 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-TRUE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -30859,8 +31897,6 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v16, v21, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB95_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB95_2
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -30877,11 +31913,16 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-FAKE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s8, s0, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -31029,8 +32070,6 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v9, v14
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v0, v15
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB95_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB95_2
; GFX11-FAKE16-NEXT: .LBB95_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -31715,7 +32754,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s72, s17, 16
; SI-NEXT: s_lshr_b32 s76, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB97_4
+; SI-NEXT: s_cbranch_scc0 .LBB97_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
@@ -31761,8 +32800,40 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[56:57], s[10:11], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 8
-; SI-NEXT: s_cbranch_execnz .LBB97_3
-; SI-NEXT: .LBB97_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB97_3
+; SI-NEXT: .LBB97_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: .LBB97_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB97_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_and_b32 s4, s22, 0xffff
; SI-NEXT: s_lshl_b32 s5, s79, 16
@@ -31827,7 +32898,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s89, s11, 24
; SI-NEXT: s_lshr_b32 s75, s11, 16
; SI-NEXT: s_lshr_b32 s88, s11, 8
-; SI-NEXT: .LBB97_3: ; %end
+; SI-NEXT: .LBB97_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s24
; SI-NEXT: v_mov_b32_e32 v2, s14
@@ -31861,38 +32932,12 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v30, s75
; SI-NEXT: v_mov_b32_e32 v31, s89
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB97_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: s_branch .LBB97_2
;
; VI-LABEL: bitcast_v16i16_to_v32i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB97_4
+; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s23, 24
; VI-NEXT: s_lshr_b32 s15, s23, 16
@@ -31918,8 +32963,40 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[6:7], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB97_3
-; VI-NEXT: .LBB97_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB97_3
+; VI-NEXT: .LBB97_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB97_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB97_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -31984,7 +33061,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b32 s57, s17, 8
; VI-NEXT: s_lshr_b32 s58, s16, 16
; VI-NEXT: s_lshr_b32 s59, s16, 8
-; VI-NEXT: .LBB97_3: ; %end
+; VI-NEXT: .LBB97_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s59
; VI-NEXT: v_mov_b32_e32 v2, s58
@@ -32018,38 +33095,12 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v30, s15
; VI-NEXT: v_mov_b32_e32 v31, s14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB97_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB97_2
;
; GFX9-LABEL: bitcast_v16i16_to_v32i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s23, 24
; GFX9-NEXT: s_lshr_b32 s15, s23, 16
@@ -32075,8 +33126,40 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB97_4
-; GFX9-NEXT: .LBB97_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB97_3
+; GFX9-NEXT: .LBB97_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB97_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s19, 3 op_sel_hi:[1,0]
@@ -32109,34 +33192,8 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX9-NEXT: s_branch .LBB97_5
-; GFX9-NEXT: .LBB97_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB97_2
-; GFX9-NEXT: .LBB97_4:
+; GFX9-NEXT: s_branch .LBB97_6
+; GFX9-NEXT: .LBB97_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -32169,7 +33226,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v19, s8
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB97_5: ; %end
+; GFX9-NEXT: .LBB97_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_mov_b32_e32 v12, v9
; GFX9-NEXT: v_mov_b32_e32 v20, v17
@@ -32185,7 +33242,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s13, s19, 24
; GFX11-NEXT: s_lshr_b32 s14, s19, 16
@@ -32211,9 +33268,40 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB97_4
-; GFX11-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB97_3
+; GFX11-NEXT: .LBB97_2:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr21
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr20
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: .LBB97_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v39, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v37, s3, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v35, s17, 3 op_sel_hi:[1,0]
@@ -32246,34 +33334,8 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB97_5
-; GFX11-NEXT: .LBB97_3:
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: s_branch .LBB97_2
-; GFX11-NEXT: .LBB97_4:
+; GFX11-NEXT: s_branch .LBB97_6
+; GFX11-NEXT: .LBB97_5:
; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
@@ -32294,7 +33356,7 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v19, s8
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB97_5: ; %end
+; GFX11-NEXT: .LBB97_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v38
; GFX11-NEXT: v_mov_b32_e32 v4, v39
; GFX11-NEXT: v_mov_b32_e32 v8, v36
@@ -33340,7 +34402,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB99_4
+; SI-NEXT: s_cbranch_scc0 .LBB99_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s8, s20, 0xff
; SI-NEXT: s_lshl_b32 s9, s21, 8
@@ -33426,8 +34488,28 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s41, s45, 16
; SI-NEXT: s_lshr_b32 s43, s88, 16
; SI-NEXT: s_lshr_b32 s13, s89, 16
-; SI-NEXT: s_cbranch_execnz .LBB99_3
-; SI-NEXT: .LBB99_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[44:45], 0
+; SI-NEXT: s_branch .LBB99_3
+; SI-NEXT: .LBB99_2:
+; SI-NEXT: s_mov_b64 s[44:45], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: .LBB99_3: ; %Flow
+; SI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; SI-NEXT: s_cselect_b32 s44, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s44, 1
+; SI-NEXT: s_cbranch_scc1 .LBB99_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_and_b32 s4, s79, 0xff
; SI-NEXT: s_lshl_b32 s5, s78, 8
@@ -33540,7 +34622,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s41, s7, 16
; SI-NEXT: s_lshr_b32 s43, s5, 16
; SI-NEXT: s_lshr_b32 s13, s9, 16
-; SI-NEXT: .LBB99_3: ; %end
+; SI-NEXT: .LBB99_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -33574,20 +34656,6 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s8
; SI-NEXT: v_mov_b32_e32 v7, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB99_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: s_branch .LBB99_2
;
; VI-LABEL: bitcast_v32i8_to_v16i16_scalar:
; VI: ; %bb.0:
@@ -33612,7 +34680,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s10, v0
-; VI-NEXT: s_cbranch_scc0 .LBB99_4
+; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -33663,8 +34731,17 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v7, s47, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB99_3
-; VI-NEXT: .LBB99_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB99_3
+; VI-NEXT: .LBB99_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB99_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB99_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: v_mov_b32_e32 v0, s46
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -33746,11 +34823,8 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB99_3: ; %end
+; VI-NEXT: .LBB99_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB99_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB99_2
;
; GFX9-LABEL: bitcast_v32i8_to_v16i16_scalar:
; GFX9: ; %bb.0:
@@ -33775,7 +34849,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
@@ -33832,8 +34906,17 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB99_3
-; GFX9-NEXT: .LBB99_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB99_3
+; GFX9-NEXT: .LBB99_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB99_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s46
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -33915,11 +34998,8 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
-; GFX9-NEXT: .LBB99_3: ; %end
+; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB99_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB99_2
;
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16i16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -33941,7 +35021,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -33984,9 +35064,17 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-TRUE16-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB99_3
+; GFX11-TRUE16-NEXT: .LBB99_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
@@ -34044,11 +35132,8 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: .LBB99_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB99_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB99_2
;
; GFX11-FAKE16-LABEL: bitcast_v32i8_to_v16i16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -34070,7 +35155,7 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -34118,9 +35203,17 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v8, 16, v7
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, s46, 16, v4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s43
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-FAKE16-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB99_3
+; GFX11-FAKE16-NEXT: .LBB99_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s41, s41, 3
; GFX11-FAKE16-NEXT: s_add_i32 s14, s14, 3
@@ -34186,11 +35279,8 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v5, 16, v10
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v9, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v7, 16, v14
-; GFX11-FAKE16-NEXT: .LBB99_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB99_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB99_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -34487,7 +35577,7 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; SI-NEXT: s_lshr_b32 s7, s17, 16
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB101_3
+; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s13, s16, 16
; SI-NEXT: s_lshl_b32 s15, s6, 16
@@ -34505,8 +35595,32 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; SI-NEXT: s_lshl_b32 s45, s12, 16
; SI-NEXT: s_lshl_b32 s46, s23, 16
; SI-NEXT: s_lshl_b32 s47, s14, 16
-; SI-NEXT: s_cbranch_execnz .LBB101_4
-; SI-NEXT: .LBB101_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s14
; SI-NEXT: v_cvt_f32_f16_e32 v1, s23
; SI-NEXT: v_cvt_f32_f16_e32 v2, s12
@@ -34571,26 +35685,8 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; SI-NEXT: s_branch .LBB101_5
-; SI-NEXT: .LBB101_3:
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: s_branch .LBB101_2
-; SI-NEXT: .LBB101_4:
+; SI-NEXT: s_branch .LBB101_6
+; SI-NEXT: .LBB101_5:
; SI-NEXT: v_mov_b32_e32 v9, s47
; SI-NEXT: v_mov_b32_e32 v8, s46
; SI-NEXT: v_mov_b32_e32 v10, s45
@@ -34607,7 +35703,7 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; SI-NEXT: v_mov_b32_e32 v2, s24
; SI-NEXT: v_mov_b32_e32 v1, s15
; SI-NEXT: v_mov_b32_e32 v0, s13
-; SI-NEXT: .LBB101_5: ; %end
+; SI-NEXT: .LBB101_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -34646,10 +35742,18 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB101_3
+; VI-NEXT: s_cbranch_scc0 .LBB101_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB101_4
-; VI-NEXT: .LBB101_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB101_3
+; VI-NEXT: .LBB101_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB101_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB101_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -34692,9 +35796,7 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; VI-NEXT: v_or_b32_e32 v1, v1, v10
; VI-NEXT: v_or_b32_e32 v0, v8, v9
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB101_3:
-; VI-NEXT: s_branch .LBB101_2
-; VI-NEXT: .LBB101_4:
+; VI-NEXT: .LBB101_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -34709,10 +35811,18 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB101_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB101_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB101_4
-; GFX9-NEXT: .LBB101_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB101_3
+; GFX9-NEXT: .LBB101_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB101_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB101_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v6, s22, v0 op_sel_hi:[1,0]
@@ -34723,9 +35833,7 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB101_3:
-; GFX9-NEXT: s_branch .LBB101_2
-; GFX9-NEXT: .LBB101_4:
+; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -34745,11 +35853,16 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; GFX11-NEXT: s_mov_b32 s4, s16
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB101_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s7 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s6 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v5, 0x200, s5 op_sel_hi:[0,1]
@@ -34759,8 +35872,6 @@ define inreg <16 x bfloat> @bitcast_v16f16_to_v16bf16_scalar(<16 x half> inreg %
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB101_3:
-; GFX11-NEXT: s_branch .LBB101_2
; GFX11-NEXT: .LBB101_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -35620,7 +36731,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v29, 1.0, s15
; SI-NEXT: v_mul_f32_e64 v27, 1.0, s11
; SI-NEXT: v_mul_f32_e64 v24, 1.0, s7
-; SI-NEXT: s_cbranch_scc0 .LBB103_4
+; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v37
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v36
@@ -35640,37 +36751,57 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v48
; SI-NEXT: v_lshr_b64 v[33:34], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[21:22], v[29:30], 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshr_b64 v[11:12], v[12:13], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
; SI-NEXT: v_lshr_b64 v[34:35], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[5:6], v[31:32], 16
; SI-NEXT: v_lshr_b64 v[22:23], v[27:28], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[24:25], 16
-; SI-NEXT: s_cbranch_execnz .LBB103_3
-; SI-NEXT: .LBB103_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB103_3
+; SI-NEXT: .LBB103_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr22
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: .LBB103_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB103_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v51
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; SI-NEXT: v_lshr_b64 v[5:6], v[1:2], 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
; SI-NEXT: v_lshr_b64 v[21:22], v[1:2], 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v49
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v27
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
; SI-NEXT: v_lshr_b64 v[22:23], v[1:2], 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; SI-NEXT: v_lshr_b64 v[25:26], v[1:2], 16
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v1
@@ -35690,21 +36821,21 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
+; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
; SI-NEXT: v_lshr_b64 v[7:8], v[15:16], 16
; SI-NEXT: v_lshr_b64 v[10:11], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
; SI-NEXT: v_lshr_b64 v[13:14], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_lshr_b64 v[33:34], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[11:12], v[12:13], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[9:10], 16
; SI-NEXT: v_lshr_b64 v[34:35], v[6:7], 16
-; SI-NEXT: .LBB103_3: ; %end
+; SI-NEXT: .LBB103_5: ; %end
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v13
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v11
@@ -35730,29 +36861,23 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB103_4:
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: s_branch .LBB103_2
;
; VI-LABEL: bitcast_v16bf16_to_v16f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB103_3
+; VI-NEXT: s_cbranch_scc0 .LBB103_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB103_4
-; VI-NEXT: .LBB103_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB103_3
+; VI-NEXT: .LBB103_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB103_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB103_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v10, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v10
@@ -35903,9 +37028,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_mov_b32_e32 v5, v13
; VI-NEXT: v_mov_b32_e32 v7, v15
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB103_3:
-; VI-NEXT: s_branch .LBB103_2
-; VI-NEXT: .LBB103_4:
+; VI-NEXT: .LBB103_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -35920,10 +37043,18 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB103_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB103_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB103_4
-; GFX9-NEXT: .LBB103_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB103_3
+; GFX9-NEXT: .LBB103_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB103_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -36079,9 +37210,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v8
; GFX9-NEXT: v_lshl_or_b32 v0, v16, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB103_3:
-; GFX9-NEXT: s_branch .LBB103_2
-; GFX9-NEXT: .LBB103_4:
+; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -36101,11 +37230,16 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s4, s16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-TRUE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -36260,8 +37394,6 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v12
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v15.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB103_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB103_2
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -36278,11 +37410,16 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_mov_b32 s4, s16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-FAKE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s8, s0, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s8
@@ -36447,8 +37584,6 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v8, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v14
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB103_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB103_2
; GFX11-FAKE16-NEXT: .LBB103_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -37140,7 +38275,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s72, s17, 16
; SI-NEXT: s_lshr_b32 s73, s16, 16
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB105_3
+; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s73, 16
@@ -37186,8 +38321,40 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: s_bfe_u32 s25, s74, 0x80008
; SI-NEXT: s_bfe_u32 s29, s76, 0x80008
; SI-NEXT: s_bfe_u32 s41, s78, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB105_4
-; SI-NEXT: .LBB105_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB105_3
+; SI-NEXT: .LBB105_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s79
; SI-NEXT: v_cvt_f32_f16_e32 v1, s22
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -37272,34 +38439,8 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[3:4], v[48:49], 24
; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 8
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v49
-; SI-NEXT: s_branch .LBB105_5
-; SI-NEXT: .LBB105_3:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr24
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: s_branch .LBB105_2
-; SI-NEXT: .LBB105_4:
+; SI-NEXT: s_branch .LBB105_6
+; SI-NEXT: .LBB105_5:
; SI-NEXT: v_mov_b32_e32 v30, s78
; SI-NEXT: v_mov_b32_e32 v22, s76
; SI-NEXT: v_mov_b32_e32 v14, s74
@@ -37332,7 +38473,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v27, s46
; SI-NEXT: v_mov_b32_e32 v24, s56
; SI-NEXT: v_mov_b32_e32 v25, s58
-; SI-NEXT: .LBB105_5: ; %end
+; SI-NEXT: .LBB105_6: ; %end
; SI-NEXT: v_mov_b32_e32 v2, v0
; SI-NEXT: v_mov_b32_e32 v0, v48
; SI-NEXT: v_mov_b32_e32 v4, v49
@@ -37351,7 +38492,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB105_3
+; VI-NEXT: s_cbranch_scc0 .LBB105_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s23, 24
; VI-NEXT: s_lshr_b32 s44, s23, 16
@@ -37377,8 +38518,40 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB105_4
-; VI-NEXT: .LBB105_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB105_3
+; VI-NEXT: .LBB105_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB105_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB105_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_add_f16_e32 v6, s4, v1
@@ -37436,34 +38609,8 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; VI-NEXT: v_bfe_u32 v23, v22, 8, 8
; VI-NEXT: v_bfe_u32 v15, v14, 8, 8
; VI-NEXT: v_bfe_u32 v7, v6, 8, 8
-; VI-NEXT: s_branch .LBB105_5
-; VI-NEXT: .LBB105_3:
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB105_2
-; VI-NEXT: .LBB105_4:
+; VI-NEXT: s_branch .LBB105_6
+; VI-NEXT: .LBB105_5:
; VI-NEXT: v_mov_b32_e32 v2, s59
; VI-NEXT: v_mov_b32_e32 v6, s58
; VI-NEXT: v_mov_b32_e32 v10, s57
@@ -37496,7 +38643,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v19, s8
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB105_5: ; %end
+; VI-NEXT: .LBB105_6: ; %end
; VI-NEXT: v_mov_b32_e32 v4, v35
; VI-NEXT: v_mov_b32_e32 v12, v34
; VI-NEXT: v_mov_b32_e32 v20, v33
@@ -37507,7 +38654,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB105_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s23, 24
; GFX9-NEXT: s_lshr_b32 s15, s23, 16
@@ -37533,8 +38680,40 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB105_4
-; GFX9-NEXT: .LBB105_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB105_3
+; GFX9-NEXT: .LBB105_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB105_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v2, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v2 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v2 op_sel_hi:[1,0]
@@ -37568,34 +38747,8 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v0
-; GFX9-NEXT: s_branch .LBB105_5
-; GFX9-NEXT: .LBB105_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB105_2
-; GFX9-NEXT: .LBB105_4:
+; GFX9-NEXT: s_branch .LBB105_6
+; GFX9-NEXT: .LBB105_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -37628,7 +38781,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v19, s8
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB105_5: ; %end
+; GFX9-NEXT: .LBB105_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_mov_b32_e32 v12, v9
; GFX9-NEXT: v_mov_b32_e32 v20, v17
@@ -37644,7 +38797,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s13, s19, 24
; GFX11-NEXT: s_lshr_b32 s14, s19, 16
@@ -37670,9 +38823,40 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB105_3
+; GFX11-NEXT: .LBB105_2:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr21
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr20
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: .LBB105_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v39, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v37, 0x200, s3 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v35, 0x200, s17 op_sel_hi:[0,1]
@@ -37705,34 +38889,8 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v39
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v38
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v38
-; GFX11-NEXT: s_branch .LBB105_5
-; GFX11-NEXT: .LBB105_3:
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr21
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr20
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: s_branch .LBB105_2
-; GFX11-NEXT: .LBB105_4:
+; GFX11-NEXT: s_branch .LBB105_6
+; GFX11-NEXT: .LBB105_5:
; GFX11-NEXT: v_dual_mov_b32 v38, s0 :: v_dual_mov_b32 v39, s1
; GFX11-NEXT: v_dual_mov_b32 v36, s2 :: v_dual_mov_b32 v37, s3
; GFX11-NEXT: v_dual_mov_b32 v34, s16 :: v_dual_mov_b32 v35, s17
@@ -37753,7 +38911,7 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-NEXT: v_mov_b32_e32 v19, s8
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB105_5: ; %end
+; GFX11-NEXT: .LBB105_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v38
; GFX11-NEXT: v_mov_b32_e32 v4, v39
; GFX11-NEXT: v_mov_b32_e32 v8, v36
@@ -38799,7 +39957,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s46, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s47, v0
-; SI-NEXT: s_cbranch_scc0 .LBB107_4
+; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s8, s20, 0xff
; SI-NEXT: s_lshl_b32 s9, s21, 8
@@ -38885,8 +40043,28 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s41, s45, 16
; SI-NEXT: s_lshr_b32 s43, s88, 16
; SI-NEXT: s_lshr_b32 s13, s89, 16
-; SI-NEXT: s_cbranch_execnz .LBB107_3
-; SI-NEXT: .LBB107_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[44:45], 0
+; SI-NEXT: s_branch .LBB107_3
+; SI-NEXT: .LBB107_2:
+; SI-NEXT: s_mov_b64 s[44:45], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: .LBB107_3: ; %Flow
+; SI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; SI-NEXT: s_cselect_b32 s44, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s44, 1
+; SI-NEXT: s_cbranch_scc1 .LBB107_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_and_b32 s4, s79, 0xff
; SI-NEXT: s_lshl_b32 s5, s78, 8
@@ -38999,7 +40177,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s41, s7, 16
; SI-NEXT: s_lshr_b32 s43, s5, 16
; SI-NEXT: s_lshr_b32 s13, s9, 16
-; SI-NEXT: .LBB107_3: ; %end
+; SI-NEXT: .LBB107_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -39033,20 +40211,6 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v6, s8
; SI-NEXT: v_mov_b32_e32 v7, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB107_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: s_branch .LBB107_2
;
; VI-LABEL: bitcast_v32i8_to_v16f16_scalar:
; VI: ; %bb.0:
@@ -39071,7 +40235,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s10, v0
-; VI-NEXT: s_cbranch_scc0 .LBB107_4
+; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -39122,8 +40286,17 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v7, s47, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB107_3
-; VI-NEXT: .LBB107_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB107_3
+; VI-NEXT: .LBB107_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB107_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB107_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: v_mov_b32_e32 v0, s46
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -39205,11 +40378,8 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB107_3: ; %end
+; VI-NEXT: .LBB107_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB107_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB107_2
;
; GFX9-LABEL: bitcast_v32i8_to_v16f16_scalar:
; GFX9: ; %bb.0:
@@ -39234,7 +40404,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
@@ -39291,8 +40461,17 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB107_3
-; GFX9-NEXT: .LBB107_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB107_3
+; GFX9-NEXT: .LBB107_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB107_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s46
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -39374,11 +40553,8 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
-; GFX9-NEXT: .LBB107_3: ; %end
+; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB107_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB107_2
;
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16f16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -39400,7 +40576,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -39443,9 +40619,17 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-TRUE16-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB107_3
+; GFX11-TRUE16-NEXT: .LBB107_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
@@ -39503,11 +40687,8 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: .LBB107_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB107_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB107_2
;
; GFX11-FAKE16-LABEL: bitcast_v32i8_to_v16f16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -39529,7 +40710,7 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -39577,9 +40758,17 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v8, 16, v7
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, s46, 16, v4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s43
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-FAKE16-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB107_3
+; GFX11-FAKE16-NEXT: .LBB107_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s41, s41, 3
; GFX11-FAKE16-NEXT: s_add_i32 s14, s14, 3
@@ -39645,11 +40834,8 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v5, 16, v10
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v9, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v7, 16, v14
-; GFX11-FAKE16-NEXT: .LBB107_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB107_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB107_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -40882,137 +42068,165 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s24, 0
; SI-NEXT: s_waitcnt expcnt(6)
; SI-NEXT: v_mul_f32_e64 v45, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s17
; SI-NEXT: s_waitcnt expcnt(4)
; SI-NEXT: v_mul_f32_e64 v47, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s15
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s15
; SI-NEXT: v_mul_f32_e64 v46, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: v_mul_f32_e64 v57, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s11
; SI-NEXT: v_mul_f32_e64 v56, 1.0, s8
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s9
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v58, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v42, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB109_4
+; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v45
-; SI-NEXT: v_lshr_b64 v[53:54], v[8:9], 16
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v45
+; SI-NEXT: v_lshr_b64 v[0:1], v[24:25], 16
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v57
+; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
+; SI-NEXT: v_lshr_b64 v[8:9], v[32:33], 16
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v46
-; SI-NEXT: v_lshr_b64 v[39:40], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[40:41], 16
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v56
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v59
-; SI-NEXT: v_lshr_b64 v[50:51], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v32
-; SI-NEXT: v_lshr_b64 v[54:55], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[40:41], v[21:22], 16
-; SI-NEXT: v_lshr_b64 v[42:43], v[24:25], 16
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v59
+; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[51:52], v[21:22], 16
+; SI-NEXT: v_lshr_b64 v[53:54], v[42:43], 16
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v58
-; SI-NEXT: v_lshr_b64 v[51:52], v[5:6], 16
-; SI-NEXT: v_lshr_b64 v[43:44], v[29:30], 16
+; SI-NEXT: v_lshr_b64 v[34:35], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[29:30], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[0:1], 8
+; SI-NEXT: v_lshr_b64 v[16:17], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[38:39], v[8:9], 8
+; SI-NEXT: v_lshr_b64 v[19:20], v[50:51], 24
+; SI-NEXT: v_lshr_b64 v[35:36], v[50:51], 16
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
+; SI-NEXT: v_lshr_b64 v[11:12], v[8:9], 24
+; SI-NEXT: v_lshr_b64 v[17:18], v[50:51], 8
+; SI-NEXT: v_lshr_b64 v[27:28], v[53:54], 24
; SI-NEXT: v_lshr_b64 v[36:37], v[53:54], 16
-; SI-NEXT: v_lshr_b64 v[9:10], v[53:54], 8
-; SI-NEXT: v_lshr_b64 v[19:20], v[39:40], 24
-; SI-NEXT: v_lshr_b64 v[34:35], v[39:40], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[50:51], 24
-; SI-NEXT: v_lshr_b64 v[48:49], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 8
-; SI-NEXT: v_lshr_b64 v[11:12], v[53:54], 24
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v32
+; SI-NEXT: v_lshr_b64 v[25:26], v[53:54], 8
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v44
; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v46
-; SI-NEXT: v_lshrrev_b32_e32 v10, 8, v51
+; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v1
; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v56
-; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v54
+; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v9
; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v58
-; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v40
-; SI-NEXT: v_lshrrev_b32_e32 v35, 8, v43
-; SI-NEXT: v_lshr_b64 v[17:18], v[39:40], 8
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 24
-; SI-NEXT: v_lshr_b64 v[37:38], v[42:43], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[42:43], 8
-; SI-NEXT: s_cbranch_execnz .LBB109_3
-; SI-NEXT: .LBB109_2: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v59
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshr_b64 v[42:43], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v29
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v58
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v57
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshr_b64 v[39:40], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v56
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v51
+; SI-NEXT: v_lshrrev_b32_e32 v49, 8, v54
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB109_3
+; SI-NEXT: .LBB109_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr20
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: .LBB109_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB109_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v59
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v42
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshr_b64 v[53:54], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v46
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshr_b64 v[53:54], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v58
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v57
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[50:51], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v56
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v32
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v8
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshr_b64 v[8:9], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; SI-NEXT: v_lshr_b64 v[54:55], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[43:44], v[29:30], 16
-; SI-NEXT: v_lshr_b64 v[40:41], v[21:22], 16
-; SI-NEXT: v_lshr_b64 v[51:52], v[5:6], 16
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v15
+; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v18
+; SI-NEXT: v_lshr_b64 v[51:52], v[21:22], 16
+; SI-NEXT: v_lshr_b64 v[9:10], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[54:55], v[29:30], 16
+; SI-NEXT: v_lshr_b64 v[34:35], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[16:17], v[8:9], 16
+; SI-NEXT: v_lshr_b64 v[35:36], v[50:51], 16
+; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 24
+; SI-NEXT: v_lshr_b64 v[48:49], v[0:1], 8
+; SI-NEXT: v_lshr_b64 v[11:12], v[8:9], 24
+; SI-NEXT: v_lshr_b64 v[38:39], v[8:9], 8
+; SI-NEXT: v_lshr_b64 v[19:20], v[50:51], 24
+; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v18
+; SI-NEXT: v_lshr_b64 v[17:18], v[50:51], 8
+; SI-NEXT: v_lshr_b64 v[27:28], v[53:54], 24
; SI-NEXT: v_lshr_b64 v[36:37], v[53:54], 16
-; SI-NEXT: v_lshr_b64 v[3:4], v[50:51], 24
-; SI-NEXT: v_lshr_b64 v[48:49], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 8
-; SI-NEXT: v_lshr_b64 v[11:12], v[53:54], 24
-; SI-NEXT: v_lshr_b64 v[9:10], v[53:54], 8
-; SI-NEXT: v_lshr_b64 v[19:20], v[39:40], 24
-; SI-NEXT: v_lshr_b64 v[34:35], v[39:40], 16
-; SI-NEXT: v_lshr_b64 v[17:18], v[39:40], 8
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 24
-; SI-NEXT: v_lshr_b64 v[37:38], v[42:43], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[42:43], 8
-; SI-NEXT: v_lshrrev_b32_e32 v10, 8, v51
-; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v54
-; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v40
-; SI-NEXT: v_lshrrev_b32_e32 v35, 8, v43
-; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v8
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v16
-; SI-NEXT: v_lshrrev_b32_e32 v31, 24, v24
-; SI-NEXT: .LBB109_3: ; %end
-; SI-NEXT: v_mov_b32_e32 v13, v20
-; SI-NEXT: v_mov_b32_e32 v20, v40
-; SI-NEXT: v_mov_b32_e32 v24, v42
-; SI-NEXT: v_mov_b32_e32 v28, v43
+; SI-NEXT: v_lshr_b64 v[25:26], v[53:54], 8
+; SI-NEXT: v_lshrrev_b32_e32 v20, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v33, 8, v9
+; SI-NEXT: v_lshrrev_b32_e32 v39, 8, v51
+; SI-NEXT: v_lshrrev_b32_e32 v49, 8, v54
+; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v15
+; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v23
+; SI-NEXT: .LBB109_5: ; %end
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -41025,52 +42239,30 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v0, v50
-; SI-NEXT: v_mov_b32_e32 v2, v48
-; SI-NEXT: v_mov_b32_e32 v4, v51
-; SI-NEXT: v_mov_b32_e32 v5, v10
-; SI-NEXT: v_mov_b32_e32 v8, v53
-; SI-NEXT: v_mov_b32_e32 v10, v36
-; SI-NEXT: v_mov_b32_e32 v12, v54
-; SI-NEXT: v_mov_b32_e32 v16, v39
-; SI-NEXT: v_mov_b32_e32 v18, v34
-; SI-NEXT: v_mov_b32_e32 v21, v33
-; SI-NEXT: v_mov_b32_e32 v26, v37
-; SI-NEXT: v_mov_b32_e32 v29, v35
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v12, v9
+; SI-NEXT: v_mov_b32_e32 v1, v48
+; SI-NEXT: v_mov_b32_e32 v2, v34
+; SI-NEXT: v_mov_b32_e32 v9, v38
+; SI-NEXT: v_mov_b32_e32 v10, v16
+; SI-NEXT: v_mov_b32_e32 v5, v20
+; SI-NEXT: v_mov_b32_e32 v13, v33
+; SI-NEXT: v_mov_b32_e32 v16, v50
+; SI-NEXT: v_mov_b32_e32 v18, v35
+; SI-NEXT: v_mov_b32_e32 v20, v51
+; SI-NEXT: v_mov_b32_e32 v21, v39
+; SI-NEXT: v_mov_b32_e32 v24, v53
+; SI-NEXT: v_mov_b32_e32 v26, v36
+; SI-NEXT: v_mov_b32_e32 v28, v54
+; SI-NEXT: v_mov_b32_e32 v29, v49
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB109_4:
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr10
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr34
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr37
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr50
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: s_branch .LBB109_2
;
; VI-LABEL: bitcast_v16bf16_to_v32i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB109_3
+; VI-NEXT: s_cbranch_scc0 .LBB109_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s57, s23, 24
; VI-NEXT: s_lshr_b32 s56, s23, 16
@@ -41096,8 +42288,40 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB109_4
-; VI-NEXT: .LBB109_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB109_3
+; VI-NEXT: .LBB109_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr25
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr24
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: .LBB109_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB109_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v2, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v2
@@ -41271,34 +42495,8 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; VI-NEXT: s_branch .LBB109_5
-; VI-NEXT: .LBB109_3:
-; VI-NEXT: ; implicit-def: $sgpr25
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr24
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: s_branch .LBB109_2
-; VI-NEXT: .LBB109_4:
+; VI-NEXT: s_branch .LBB109_6
+; VI-NEXT: .LBB109_5:
; VI-NEXT: v_mov_b32_e32 v26, s59
; VI-NEXT: v_mov_b32_e32 v25, s58
; VI-NEXT: v_mov_b32_e32 v31, s57
@@ -41331,7 +42529,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v37, s8
; VI-NEXT: v_mov_b32_e32 v34, s6
; VI-NEXT: v_mov_b32_e32 v32, s4
-; VI-NEXT: .LBB109_5: ; %end
+; VI-NEXT: .LBB109_6: ; %end
; VI-NEXT: v_mov_b32_e32 v3, v32
; VI-NEXT: v_mov_b32_e32 v11, v34
; VI-NEXT: v_mov_b32_e32 v19, v37
@@ -41342,7 +42540,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB109_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s29, s23, 24
; GFX9-NEXT: s_lshr_b32 s58, s23, 16
@@ -41368,8 +42566,40 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB109_4
-; GFX9-NEXT: .LBB109_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB109_3
+; GFX9-NEXT: .LBB109_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr25
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr24
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: .LBB109_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s17, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -41547,34 +42777,8 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
-; GFX9-NEXT: s_branch .LBB109_5
-; GFX9-NEXT: .LBB109_3:
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr25
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr24
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: s_branch .LBB109_2
-; GFX9-NEXT: .LBB109_4:
+; GFX9-NEXT: s_branch .LBB109_6
+; GFX9-NEXT: .LBB109_5:
; GFX9-NEXT: v_mov_b32_e32 v24, s22
; GFX9-NEXT: v_mov_b32_e32 v26, s59
; GFX9-NEXT: v_mov_b32_e32 v32, s23
@@ -41607,7 +42811,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s8
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB109_5: ; %end
+; GFX9-NEXT: .LBB109_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v4, v35
; GFX9-NEXT: v_mov_b32_e32 v12, v34
; GFX9-NEXT: v_mov_b32_e32 v20, v33
@@ -41619,7 +42823,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b32 s24, s19, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s19, 16
@@ -41645,189 +42849,9 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-TRUE16-NEXT: .LBB109_2: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xffff0000
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, 0xffff0000
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, 0xffff0000
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, 0xffff0000
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v9, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v10, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s18, 16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v35.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, v13, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x7fff, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v7
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s17, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v34.l
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s16, 0xffff0000
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v7 :: v_dual_add_nc_u32 v4, v4, v9
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s16, 16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s19, 0xffff0000
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v5 :: v_dual_add_nc_u32 v7, v11, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s19, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s18, 0xffff0000
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v33.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v18.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v32.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v26.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[50:51]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v50
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v36
-; GFX11-TRUE16-NEXT: s_branch .LBB109_5
-; GFX11-TRUE16-NEXT: .LBB109_3:
+; GFX11-TRUE16-NEXT: s_branch .LBB109_3
+; GFX11-TRUE16-NEXT: .LBB109_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr22
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
@@ -41852,8 +42876,193 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr26
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr24
-; GFX11-TRUE16-NEXT: s_branch .LBB109_2
-; GFX11-TRUE16-NEXT: .LBB109_4:
+; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xffff0000
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, 0xffff0000
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, 0xffff0000
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s3
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, 0xffff0000
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v9, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v10, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s18, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v35.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, v13, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x7fff, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s17, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v34.l
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s16, 0xffff0000
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v7 :: v_dual_add_nc_u32 v4, v4, v9
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s16, 16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s19, 0xffff0000
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v5 :: v_dual_add_nc_u32 v7, v11, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s19, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v9, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s18, 0xffff0000
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v33.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v22.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v32.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v26.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v48
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[50:51]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v50
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v36
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
+; GFX11-TRUE16-NEXT: .LBB109_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v33, s17
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s46 :: v_dual_mov_b32 v35, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s19 :: v_dual_mov_b32 v25, s28
@@ -41872,7 +43081,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v35
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v34
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v20, v33
@@ -41884,7 +43093,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b32 s24, s19, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s19, 16
@@ -41910,9 +43119,40 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-FAKE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB109_3
+; GFX11-FAKE16-NEXT: .LBB109_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
+; GFX11-FAKE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
@@ -42095,34 +43335,8 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[50:51]
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
-; GFX11-FAKE16-NEXT: s_branch .LBB109_5
-; GFX11-FAKE16-NEXT: .LBB109_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr22
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr29
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr25
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr20
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr27
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr23
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr21
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr26
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr24
-; GFX11-FAKE16-NEXT: s_branch .LBB109_2
-; GFX11-FAKE16-NEXT: .LBB109_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB109_6
+; GFX11-FAKE16-NEXT: .LBB109_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v33, s17
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s46 :: v_dual_mov_b32 v35, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s19 :: v_dual_mov_b32 v25, s28
@@ -42141,7 +43355,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v19, s8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-FAKE16-NEXT: .LBB109_5: ; %end
+; GFX11-FAKE16-NEXT: .LBB109_6: ; %end
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v35
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v12, v34
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v20, v33
@@ -43187,7 +44401,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB111_4
+; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -43253,8 +44467,32 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s63, 24
; SI-NEXT: s_or_b32 s79, s5, s4
-; SI-NEXT: s_cbranch_execnz .LBB111_3
-; SI-NEXT: .LBB111_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB111_3
+; SI-NEXT: .LBB111_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: .LBB111_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB111_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s78, s78, 3
; SI-NEXT: s_and_b32 s4, s78, 0xff
; SI-NEXT: s_lshl_b32 s5, s77, 8
@@ -43375,7 +44613,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; SI-NEXT: s_lshl_b32 s62, s5, 16
; SI-NEXT: s_and_b32 s79, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s75, s4, 16
-; SI-NEXT: .LBB111_3: ; %end
+; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
@@ -43409,24 +44647,6 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s75
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB111_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: s_branch .LBB111_2
;
; VI-LABEL: bitcast_v32i8_to_v16bf16_scalar:
; VI: ; %bb.0:
@@ -43451,7 +44671,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s10, v0
-; VI-NEXT: s_cbranch_scc0 .LBB111_4
+; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v7, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -43502,8 +44722,17 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; VI-NEXT: v_perm_b32 v7, s47, v9, v7
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: s_cbranch_execnz .LBB111_3
-; VI-NEXT: .LBB111_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB111_3
+; VI-NEXT: .LBB111_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; VI-NEXT: .LBB111_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB111_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: v_mov_b32_e32 v0, s46
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -43585,11 +44814,8 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x3000000, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x3000000, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
-; VI-NEXT: .LBB111_3: ; %end
+; VI-NEXT: .LBB111_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB111_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; VI-NEXT: s_branch .LBB111_2
;
; GFX9-LABEL: bitcast_v32i8_to_v16bf16_scalar:
; GFX9: ; %bb.0:
@@ -43614,7 +44840,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc0c0004
@@ -43671,8 +44897,17 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v7, s56, 16, v4
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v6, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB111_3
-; GFX9-NEXT: .LBB111_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB111_3
+; GFX9-NEXT: .LBB111_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX9-NEXT: .LBB111_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s47, s47, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s46
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -43754,11 +44989,8 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v3, v9, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v4, v8, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v7, v15, 16, v7
-; GFX9-NEXT: .LBB111_3: ; %end
+; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB111_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX9-NEXT: s_branch .LBB111_2
;
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16bf16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -43780,7 +45012,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -43823,9 +45055,17 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s43
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s44
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-TRUE16-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB111_3
+; GFX11-TRUE16-NEXT: .LBB111_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
@@ -43883,11 +45123,8 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: .LBB111_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB111_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB111_2
;
; GFX11-FAKE16-LABEL: bitcast_v32i8_to_v16bf16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -43909,7 +45146,7 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s43, s28, 0xff
@@ -43957,9 +45194,17 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v8, 16, v7
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, s46, 16, v4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s43
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-FAKE16-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB111_3
+; GFX11-FAKE16-NEXT: .LBB111_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s41, s41, 3
; GFX11-FAKE16-NEXT: s_add_i32 s14, s14, 3
@@ -44025,11 +45270,8 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v5, 16, v10
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v9, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v7, 16, v14
-; GFX11-FAKE16-NEXT: .LBB111_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB111_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB111_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
index ead5d76b2e572..96a513fa91aee 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.288bit.ll
@@ -114,10 +114,18 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s22, s22, 3
@@ -127,7 +135,7 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -138,17 +146,23 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v9i32_to_v9f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
@@ -158,7 +172,7 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -169,17 +183,23 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v9i32_to_v9f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
@@ -189,7 +209,7 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -200,19 +220,22 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v9i32_to_v9f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
@@ -222,7 +245,7 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -230,8 +253,6 @@ define inreg <9 x float> @bitcast_v9i32_to_v9f32_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_mov_b32_e32 v8, s20
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -352,10 +373,18 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
; SI-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -366,9 +395,7 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -384,10 +411,18 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -398,9 +433,7 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -416,10 +449,18 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -430,9 +471,7 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -453,11 +492,16 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s18, 1.0
@@ -468,8 +512,6 @@ define inreg <9 x i32> @bitcast_v9f32_to_v9i32_scalar(<9 x float> inreg %a, i32
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -659,7 +701,7 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s26, s21, 16
@@ -670,8 +712,25 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s17, s17, 3
@@ -690,7 +749,7 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s27, s19, 16
; SI-NEXT: s_lshr_b32 s28, s17, 16
; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -728,26 +787,23 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v7, s12
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v9i32_to_v18i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
@@ -757,7 +813,7 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -768,17 +824,23 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v9i32_to_v18i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
@@ -788,7 +850,7 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -799,19 +861,22 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v9i32_to_v18i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
@@ -821,7 +886,7 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -829,8 +894,6 @@ define inreg <18 x i16> @bitcast_v9i32_to_v18i16_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_mov_b32_e32 v8, s20
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1085,7 +1148,7 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s42, s17, 16
; SI-NEXT: s_lshr_b32 s43, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -1114,8 +1177,17 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s12, s24, 0xffff
; SI-NEXT: s_lshl_b32 s14, s13, 16
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -1161,7 +1233,7 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1172,18 +1244,23 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v18i16_to_v9i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s24, 3
; VI-NEXT: s_and_b32 s4, s24, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1229,7 +1306,7 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1240,17 +1317,23 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v18i16_to_v9i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
@@ -1261,9 +1344,7 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1284,11 +1365,16 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s18, 3 op_sel_hi:[1,0]
@@ -1299,8 +1385,6 @@ define inreg <9 x i32> @bitcast_v18i16_to_v9i32_scalar(<18 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -1490,7 +1574,7 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s23, 16
; SI-NEXT: s_lshr_b32 s26, s21, 16
@@ -1501,8 +1585,25 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s17, s17, 3
@@ -1521,7 +1622,7 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b32 s27, s19, 16
; SI-NEXT: s_lshr_b32 s28, s17, 16
; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1559,26 +1660,23 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v7, s12
; SI-NEXT: v_mov_b32_e32 v8, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v9i32_to_v18f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s22, s22, 3
@@ -1588,7 +1686,7 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1599,17 +1697,23 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v9i32_to_v18f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
; GFX9-NEXT: s_add_i32 s22, s22, 3
@@ -1619,7 +1723,7 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1630,19 +1734,22 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v7, s23
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v9i32_to_v18f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
; GFX11-NEXT: s_add_i32 s18, s18, 3
@@ -1652,7 +1759,7 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1660,8 +1767,6 @@ define inreg <18 x half> @bitcast_v9i32_to_v18f16_scalar(<9 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_mov_b32_e32 v8, s20
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1953,7 +2058,7 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s42, s17, 16
; SI-NEXT: s_lshr_b32 s43, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -1982,8 +2087,17 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s12, s24, 0xffff
; SI-NEXT: s_lshl_b32 s14, s13, 16
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s43
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s42
@@ -2057,10 +2171,7 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2076,10 +2187,18 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s24, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2127,9 +2246,7 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v9
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2145,10 +2262,18 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
@@ -2160,9 +2285,7 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2183,11 +2306,16 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s18 op_sel_hi:[0,1]
@@ -2198,8 +2326,6 @@ define inreg <9 x i32> @bitcast_v18f16_to_v9i32_scalar(<18 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2384,7 +2510,7 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_3
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s17, 16
; SI-NEXT: s_lshr_b32 s28, s23, 16
@@ -2395,8 +2521,25 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_4
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -2415,19 +2558,8 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v1
-; SI-NEXT: s_branch .LBB13_5
-; SI-NEXT: .LBB13_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB13_2
-; SI-NEXT: .LBB13_4:
+; SI-NEXT: s_branch .LBB13_6
+; SI-NEXT: .LBB13_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2446,7 +2578,7 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v13, s10
; SI-NEXT: v_mov_b32_e32 v12, s6
; SI-NEXT: v_mov_b32_e32 v9, s4
-; SI-NEXT: .LBB13_5: ; %end
+; SI-NEXT: .LBB13_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v11
@@ -2480,10 +2612,18 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_3
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_4
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -2494,9 +2634,7 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_3:
-; VI-NEXT: s_branch .LBB13_2
-; VI-NEXT: .LBB13_4:
+; VI-NEXT: .LBB13_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2519,10 +2657,18 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_4
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -2533,9 +2679,7 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_3:
-; GFX9-NEXT: s_branch .LBB13_2
-; GFX9-NEXT: .LBB13_4:
+; GFX9-NEXT: .LBB13_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2563,11 +2707,16 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_4
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s18, 1.0
@@ -2578,8 +2727,6 @@ define inreg <18 x i16> @bitcast_v9f32_to_v18i16_scalar(<9 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_3:
-; GFX11-NEXT: s_branch .LBB13_2
; GFX11-NEXT: .LBB13_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2844,7 +2991,7 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s42, s17, 16
; SI-NEXT: s_lshr_b32 s43, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -2873,8 +3020,17 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s12, s24, 0xffff
; SI-NEXT: s_lshl_b32 s14, s13, 16
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -2920,7 +3076,7 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s10, s10, 0x30000
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2931,18 +3087,23 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v7, s11
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v18i16_to_v9f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s24, 3
; VI-NEXT: s_and_b32 s4, s24, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2988,7 +3149,7 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2999,17 +3160,23 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v18i16_to_v9f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
@@ -3020,9 +3187,7 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3043,11 +3208,16 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s18, 3 op_sel_hi:[1,0]
@@ -3058,8 +3228,6 @@ define inreg <9 x float> @bitcast_v18i16_to_v9f32_scalar(<18 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3244,7 +3412,7 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_3
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s25, s17, 16
; SI-NEXT: s_lshr_b32 s28, s23, 16
@@ -3255,8 +3423,25 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; SI-NEXT: s_lshr_b64 s[6:7], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_4
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -3275,19 +3460,8 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v1
-; SI-NEXT: s_branch .LBB17_5
-; SI-NEXT: .LBB17_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB17_2
-; SI-NEXT: .LBB17_4:
+; SI-NEXT: s_branch .LBB17_6
+; SI-NEXT: .LBB17_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3306,7 +3480,7 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v13, s10
; SI-NEXT: v_mov_b32_e32 v12, s6
; SI-NEXT: v_mov_b32_e32 v9, s4
-; SI-NEXT: .LBB17_5: ; %end
+; SI-NEXT: .LBB17_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v11
@@ -3340,10 +3514,18 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_3
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_4
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
; VI-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -3354,9 +3536,7 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_3:
-; VI-NEXT: s_branch .LBB17_2
-; VI-NEXT: .LBB17_4:
+; VI-NEXT: .LBB17_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3379,10 +3559,18 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_4
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
; GFX9-NEXT: v_add_f32_e64 v6, s22, 1.0
@@ -3393,9 +3581,7 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_3:
-; GFX9-NEXT: s_branch .LBB17_2
-; GFX9-NEXT: .LBB17_4:
+; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3423,11 +3609,16 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_4
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s18, 1.0
@@ -3438,8 +3629,6 @@ define inreg <18 x half> @bitcast_v9f32_to_v18f16_scalar(<9 x float> inreg %a, i
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_3:
-; GFX11-NEXT: s_branch .LBB17_2
; GFX11-NEXT: .LBB17_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3741,7 +3930,7 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s42, s17, 16
; SI-NEXT: s_lshr_b32 s43, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s43, 16
@@ -3770,8 +3959,17 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; SI-NEXT: s_and_b32 s12, s24, 0xffff
; SI-NEXT: s_lshl_b32 s14, s13, 16
; SI-NEXT: s_or_b32 s12, s12, s14
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s43
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s42
@@ -3845,10 +4043,7 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -3864,10 +4059,18 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s24, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -3915,9 +4118,7 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v9
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3933,10 +4134,18 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
@@ -3948,9 +4157,7 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3971,11 +4178,16 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s18 op_sel_hi:[0,1]
@@ -3986,8 +4198,6 @@ define inreg <9 x float> @bitcast_v18f16_to_v9f32_scalar(<18 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4288,7 +4498,7 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s43, s17, 16
; SI-NEXT: s_lshr_b32 s47, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s24, 0xffff
; SI-NEXT: s_lshl_b32 s7, s42, 16
@@ -4322,11 +4532,28 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[26:27], s[62:63], 16
; SI-NEXT: s_lshr_b64 s[28:29], s[72:73], 16
; SI-NEXT: s_mov_b32 s11, s41
+; SI-NEXT: s_mov_b64 s[40:41], 0
; SI-NEXT: s_mov_b32 s9, s61
; SI-NEXT: s_mov_b32 s7, s63
; SI-NEXT: s_mov_b32 s5, s73
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[40:41], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[40:41], s[40:41], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_and_b32 s4, s22, 0xffff
; SI-NEXT: s_lshl_b32 s5, s58, 16
@@ -4381,7 +4608,7 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s45, s7, 16
; SI-NEXT: s_lshr_b32 s46, s5, 16
; SI-NEXT: s_lshr_b32 s42, s25, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -4419,26 +4646,23 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v7, s5
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v18i16_to_v18f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_add_i32 s7, s17, 3
@@ -4484,7 +4708,7 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4495,17 +4719,23 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v7, s23
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v18i16_to_v18f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v6, s22, 3 op_sel_hi:[1,0]
@@ -4516,9 +4746,7 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4546,11 +4774,16 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s18, 3 op_sel_hi:[1,0]
@@ -4561,8 +4794,6 @@ define inreg <18 x half> @bitcast_v18i16_to_v18f16_scalar(<18 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4833,10 +5064,18 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s14, s16, 16
; SI-NEXT: s_cmp_lg_u32 s25, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s14
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s13
@@ -4913,10 +5152,8 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[12:13], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[10:11], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[6:7], 16
-; SI-NEXT: s_branch .LBB23_5
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v17, s11
; SI-NEXT: v_mov_b32_e32 v21, s9
; SI-NEXT: v_mov_b32_e32 v22, s8
@@ -4935,7 +5172,7 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s13
; SI-NEXT: v_mov_b32_e32 v10, s12
; SI-NEXT: v_mov_b32_e32 v8, s10
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: .LBB23_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v14
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -4969,10 +5206,18 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s25, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s23, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s5
@@ -5020,9 +5265,7 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v0, v1
; VI-NEXT: v_or_b32_e32 v0, v9, v10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5045,10 +5288,18 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s25, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v7, s23, v0 op_sel_hi:[1,0]
@@ -5060,9 +5311,7 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5090,11 +5339,16 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s21, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s18 op_sel_hi:[0,1]
@@ -5105,8 +5359,6 @@ define inreg <18 x i16> @bitcast_v18f16_to_v18i16_scalar(<18 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index aba9755a4f357..826e745ed1f60 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -121,10 +121,18 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s23, s23, 3
@@ -135,7 +143,7 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -147,17 +155,23 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v10i32_to_v10f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
@@ -168,7 +182,7 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -180,17 +194,23 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v10i32_to_v10f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
@@ -201,7 +221,7 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -213,19 +233,22 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v10i32_to_v10f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
@@ -236,7 +259,7 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -244,8 +267,6 @@ define inreg <10 x float> @bitcast_v10i32_to_v10f32_scalar(<10 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -372,10 +393,18 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -387,9 +416,7 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -406,10 +433,18 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -421,9 +456,7 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -440,10 +473,18 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -455,9 +496,7 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -479,11 +518,16 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
@@ -495,8 +539,6 @@ define inreg <10 x i32> @bitcast_v10f32_to_v10i32_scalar(<10 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -699,7 +741,7 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s26, s25, 16
; SI-NEXT: s_lshr_b32 s27, s23, 16
@@ -711,8 +753,26 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -733,7 +793,7 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s28, s21, 16
; SI-NEXT: s_lshr_b32 s29, s19, 16
; SI-NEXT: s_lshr_b32 s40, s17, 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -775,27 +835,23 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v10i32_to_v20i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
@@ -806,7 +862,7 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -818,17 +874,23 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v10i32_to_v20i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
@@ -839,7 +901,7 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -851,19 +913,22 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v10i32_to_v20i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
@@ -874,7 +939,7 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -882,8 +947,6 @@ define inreg <20 x i16> @bitcast_v10i32_to_v20i16_scalar(<10 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1156,7 +1219,7 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_lshr_b32 s46, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -1188,8 +1251,17 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s13, s25, 0xffff
; SI-NEXT: s_lshl_b32 s14, s27, 16
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -1240,7 +1312,7 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1252,18 +1324,23 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v20i16_to_v10i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s25, 3
; VI-NEXT: s_and_b32 s4, s25, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1314,7 +1391,7 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1326,17 +1403,23 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v20i16_to_v10i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
@@ -1348,9 +1431,7 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1372,11 +1453,16 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
@@ -1388,8 +1474,6 @@ define inreg <10 x i32> @bitcast_v20i16_to_v10i32_scalar(<20 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -1592,7 +1676,7 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s26, s25, 16
; SI-NEXT: s_lshr_b32 s27, s23, 16
@@ -1604,8 +1688,26 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1626,7 +1728,7 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s28, s21, 16
; SI-NEXT: s_lshr_b32 s29, s19, 16
; SI-NEXT: s_lshr_b32 s40, s17, 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1668,27 +1770,23 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v10i32_to_v20f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
@@ -1699,7 +1797,7 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1711,17 +1809,23 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v10i32_to_v20f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
@@ -1732,7 +1836,7 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1744,19 +1848,22 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v10i32_to_v20f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
@@ -1767,7 +1874,7 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1775,8 +1882,6 @@ define inreg <20 x half> @bitcast_v10i32_to_v20f16_scalar(<10 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2089,7 +2194,7 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_lshr_b32 s46, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -2121,8 +2226,17 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; SI-NEXT: s_and_b32 s13, s25, 0xffff
; SI-NEXT: s_lshl_b32 s14, s27, 16
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s46
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s45
@@ -2204,10 +2318,7 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2224,10 +2335,18 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s25, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2280,9 +2399,7 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2299,10 +2416,18 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
@@ -2315,9 +2440,7 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2339,11 +2462,16 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
@@ -2355,8 +2483,6 @@ define inreg <10 x i32> @bitcast_v20f16_to_v10i32_scalar(<20 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3256,7 +3382,7 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s25, 24
; SI-NEXT: s_lshr_b32 s73, s25, 16
@@ -3288,8 +3414,46 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3330,7 +3494,7 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s5, s60, 8
; SI-NEXT: s_and_b32 s7, s16, 0xff
; SI-NEXT: s_or_b32 s5, s7, s5
@@ -3460,44 +3624,12 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v10i32_to_v40i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s26, s25, 24
; VI-NEXT: s_lshr_b32 s27, s25, 16
@@ -3529,8 +3661,46 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s19, s19, 3
@@ -3571,7 +3741,7 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; VI-NEXT: s_lshr_b32 s74, s17, 8
; VI-NEXT: s_lshr_b32 s75, s16, 16
; VI-NEXT: s_lshr_b32 s76, s16, 8
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: v_mov_b32_e32 v1, s76
@@ -3654,44 +3824,12 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v10i32_to_v40i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s26, s25, 24
; GFX9-NEXT: s_lshr_b32 s27, s25, 16
@@ -3723,123 +3861,10 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
-; GFX9-NEXT: s_add_i32 s17, s17, 3
-; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: s_add_i32 s19, s19, 3
-; GFX9-NEXT: s_add_i32 s18, s18, 3
-; GFX9-NEXT: s_add_i32 s21, s21, 3
-; GFX9-NEXT: s_add_i32 s20, s20, 3
-; GFX9-NEXT: s_add_i32 s23, s23, 3
-; GFX9-NEXT: s_add_i32 s22, s22, 3
-; GFX9-NEXT: s_add_i32 s25, s25, 3
-; GFX9-NEXT: s_add_i32 s24, s24, 3
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[24:25], 24
-; GFX9-NEXT: s_lshr_b64 s[6:7], s[22:23], 24
-; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
-; GFX9-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
-; GFX9-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; GFX9-NEXT: s_lshr_b32 s26, s25, 24
-; GFX9-NEXT: s_lshr_b32 s27, s25, 16
-; GFX9-NEXT: s_lshr_b32 s28, s25, 8
-; GFX9-NEXT: s_lshr_b32 s29, s24, 16
-; GFX9-NEXT: s_lshr_b32 s40, s24, 8
-; GFX9-NEXT: s_lshr_b32 s41, s23, 24
-; GFX9-NEXT: s_lshr_b32 s42, s23, 16
-; GFX9-NEXT: s_lshr_b32 s43, s23, 8
-; GFX9-NEXT: s_lshr_b32 s44, s22, 16
-; GFX9-NEXT: s_lshr_b32 s45, s22, 8
-; GFX9-NEXT: s_lshr_b32 s46, s21, 24
-; GFX9-NEXT: s_lshr_b32 s47, s21, 16
-; GFX9-NEXT: s_lshr_b32 s56, s21, 8
-; GFX9-NEXT: s_lshr_b32 s57, s20, 16
-; GFX9-NEXT: s_lshr_b32 s58, s20, 8
-; GFX9-NEXT: s_lshr_b32 s59, s19, 24
-; GFX9-NEXT: s_lshr_b32 s60, s19, 16
-; GFX9-NEXT: s_lshr_b32 s61, s19, 8
-; GFX9-NEXT: s_lshr_b32 s62, s18, 16
-; GFX9-NEXT: s_lshr_b32 s63, s18, 8
-; GFX9-NEXT: s_lshr_b32 s72, s17, 24
-; GFX9-NEXT: s_lshr_b32 s73, s17, 16
-; GFX9-NEXT: s_lshr_b32 s74, s17, 8
-; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_lshr_b32 s76, s16, 8
-; GFX9-NEXT: .LBB13_3: ; %end
-; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
-; GFX9-NEXT: v_mov_b32_e32 v1, s76
-; GFX9-NEXT: v_perm_b32 v3, s75, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s16, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s72
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; GFX9-NEXT: v_mov_b32_e32 v1, s74
-; GFX9-NEXT: v_perm_b32 v3, s73, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s17, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s10
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT: v_mov_b32_e32 v1, s63
-; GFX9-NEXT: v_perm_b32 v3, s62, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s18, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s59
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
-; GFX9-NEXT: v_mov_b32_e32 v1, s61
-; GFX9-NEXT: v_perm_b32 v3, s60, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s19, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s8
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
-; GFX9-NEXT: v_mov_b32_e32 v1, s58
-; GFX9-NEXT: v_perm_b32 v3, s57, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s20, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s46
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
-; GFX9-NEXT: v_mov_b32_e32 v1, s56
-; GFX9-NEXT: v_perm_b32 v3, s47, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s21, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_mov_b32_e32 v1, s45
-; GFX9-NEXT: v_perm_b32 v3, s44, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s22, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s41
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
-; GFX9-NEXT: v_mov_b32_e32 v1, s43
-; GFX9-NEXT: v_perm_b32 v3, s42, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s23, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: v_mov_b32_e32 v1, s40
-; GFX9-NEXT: v_perm_b32 v3, s29, v3, v2
-; GFX9-NEXT: v_perm_b32 v1, s24, v1, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
-; GFX9-NEXT: v_mov_b32_e32 v1, s28
-; GFX9-NEXT: v_mov_b32_e32 v3, s26
-; GFX9-NEXT: v_perm_b32 v1, s25, v1, v2
-; GFX9-NEXT: v_perm_b32 v2, s27, v3, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
; GFX9-NEXT: ; implicit-def: $sgpr76
; GFX9-NEXT: ; implicit-def: $sgpr75
; GFX9-NEXT: ; implicit-def: $sgpr12
@@ -3870,14 +3895,133 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; GFX9-NEXT: ; implicit-def: $sgpr28
; GFX9-NEXT: ; implicit-def: $sgpr27
; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: s_branch .LBB13_2
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_add_i32 s17, s17, 3
+; GFX9-NEXT: s_add_i32 s16, s16, 3
+; GFX9-NEXT: s_add_i32 s19, s19, 3
+; GFX9-NEXT: s_add_i32 s18, s18, 3
+; GFX9-NEXT: s_add_i32 s21, s21, 3
+; GFX9-NEXT: s_add_i32 s20, s20, 3
+; GFX9-NEXT: s_add_i32 s23, s23, 3
+; GFX9-NEXT: s_add_i32 s22, s22, 3
+; GFX9-NEXT: s_add_i32 s25, s25, 3
+; GFX9-NEXT: s_add_i32 s24, s24, 3
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[24:25], 24
+; GFX9-NEXT: s_lshr_b64 s[6:7], s[22:23], 24
+; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
+; GFX9-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
+; GFX9-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
+; GFX9-NEXT: s_lshr_b32 s26, s25, 24
+; GFX9-NEXT: s_lshr_b32 s27, s25, 16
+; GFX9-NEXT: s_lshr_b32 s28, s25, 8
+; GFX9-NEXT: s_lshr_b32 s29, s24, 16
+; GFX9-NEXT: s_lshr_b32 s40, s24, 8
+; GFX9-NEXT: s_lshr_b32 s41, s23, 24
+; GFX9-NEXT: s_lshr_b32 s42, s23, 16
+; GFX9-NEXT: s_lshr_b32 s43, s23, 8
+; GFX9-NEXT: s_lshr_b32 s44, s22, 16
+; GFX9-NEXT: s_lshr_b32 s45, s22, 8
+; GFX9-NEXT: s_lshr_b32 s46, s21, 24
+; GFX9-NEXT: s_lshr_b32 s47, s21, 16
+; GFX9-NEXT: s_lshr_b32 s56, s21, 8
+; GFX9-NEXT: s_lshr_b32 s57, s20, 16
+; GFX9-NEXT: s_lshr_b32 s58, s20, 8
+; GFX9-NEXT: s_lshr_b32 s59, s19, 24
+; GFX9-NEXT: s_lshr_b32 s60, s19, 16
+; GFX9-NEXT: s_lshr_b32 s61, s19, 8
+; GFX9-NEXT: s_lshr_b32 s62, s18, 16
+; GFX9-NEXT: s_lshr_b32 s63, s18, 8
+; GFX9-NEXT: s_lshr_b32 s72, s17, 24
+; GFX9-NEXT: s_lshr_b32 s73, s17, 16
+; GFX9-NEXT: s_lshr_b32 s74, s17, 8
+; GFX9-NEXT: s_lshr_b32 s75, s16, 16
+; GFX9-NEXT: s_lshr_b32 s76, s16, 8
+; GFX9-NEXT: .LBB13_5: ; %end
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s76
+; GFX9-NEXT: v_perm_b32 v3, s75, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s16, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s72
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: v_mov_b32_e32 v1, s74
+; GFX9-NEXT: v_perm_b32 v3, s73, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s17, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: v_mov_b32_e32 v1, s63
+; GFX9-NEXT: v_perm_b32 v3, s62, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s18, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s59
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
+; GFX9-NEXT: v_mov_b32_e32 v1, s61
+; GFX9-NEXT: v_perm_b32 v3, s60, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s19, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
+; GFX9-NEXT: v_mov_b32_e32 v1, s58
+; GFX9-NEXT: v_perm_b32 v3, s57, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s20, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s46
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
+; GFX9-NEXT: v_mov_b32_e32 v1, s56
+; GFX9-NEXT: v_perm_b32 v3, s47, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s21, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
+; GFX9-NEXT: v_mov_b32_e32 v1, s45
+; GFX9-NEXT: v_perm_b32 v3, s44, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s22, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s41
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
+; GFX9-NEXT: v_mov_b32_e32 v1, s43
+; GFX9-NEXT: v_perm_b32 v3, s42, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s23, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:28
+; GFX9-NEXT: v_mov_b32_e32 v1, s40
+; GFX9-NEXT: v_perm_b32 v3, s29, v3, v2
+; GFX9-NEXT: v_perm_b32 v1, s24, v1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
+; GFX9-NEXT: v_mov_b32_e32 v1, s28
+; GFX9-NEXT: v_mov_b32_e32 v3, s26
+; GFX9-NEXT: v_perm_b32 v1, s25, v1, v2
+; GFX9-NEXT: v_perm_b32 v2, s27, v3, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: bitcast_v10i32_to_v40i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s63, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s21, 24
; GFX11-NEXT: s_lshr_b32 s22, s21, 16
@@ -3909,9 +4053,46 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[10:11], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[12:13], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s63
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s63, -1
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s63, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s3, s3, 3
@@ -3952,7 +4133,7 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s60, s1, 8
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
; GFX11-NEXT: s_lshr_b32 s62, s0, 8
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: v_mov_b32_e32 v6, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v7, s56, s10, v6
@@ -4000,38 +4181,6 @@ define inreg <40 x i8> @bitcast_v10i32_to_v40i8_scalar(<10 x i32> inreg %a, i32
; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
; GFX11-NEXT: scratch_store_b64 v0, v[9:10], off offset:32
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5279,7 +5428,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s79, v2
; SI-NEXT: v_readfirstlane_b32 s88, v1
; SI-NEXT: v_readfirstlane_b32 s89, v0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -5371,8 +5520,17 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -5503,7 +5661,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -5515,9 +5673,6 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v40i8_to_v10i32_scalar:
; VI: ; %bb.0:
@@ -5550,7 +5705,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s62, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s63, v0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -5613,8 +5768,17 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v9, s7, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -5716,11 +5880,8 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v40i8_to_v10i32_scalar:
; GFX9: ; %bb.0:
@@ -5753,7 +5914,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s62, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -5816,8 +5977,17 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v9, s7, v11, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: v_or_b32_e32 v9, v10, v9
-; GFX9-NEXT: s_cbranch_execnz .LBB15_3
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -5910,11 +6080,8 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v10
; GFX9-NEXT: v_add_u32_sdwa v9, v9, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB15_3: ; %end
+; GFX9-NEXT: .LBB15_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX9-NEXT: s_branch .LBB15_2
;
; GFX11-LABEL: bitcast_v40i8_to_v10i32_scalar:
; GFX11: ; %bb.0:
@@ -5944,7 +6111,7 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s57, v0
; GFX11-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-NEXT: s_mov_b32 s58, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB15_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5988,9 +6155,17 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v7, v12, v10
; GFX11-NEXT: v_or_b32_e32 v8, v14, v11
; GFX11-NEXT: v_or_b32_e32 v9, v15, v13
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_3
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB15_3
+; GFX11-NEXT: .LBB15_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-NEXT: .LBB15_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -6082,11 +6257,8 @@ define inreg <10 x i32> @bitcast_v40i8_to_v10i32_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_or_b32_e32 v8, v12, v13
; GFX11-NEXT: v_or_b32_e32 v9, v14, v15
-; GFX11-NEXT: .LBB15_3: ; %end
+; GFX11-NEXT: .LBB15_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX11-NEXT: s_branch .LBB15_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6219,10 +6391,18 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s23, s23, 3
@@ -6233,7 +6413,7 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6245,17 +6425,23 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v10i32_to_v5f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
@@ -6266,7 +6452,7 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6278,17 +6464,23 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v10i32_to_v5f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
@@ -6299,7 +6491,7 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6311,19 +6503,22 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v10i32_to_v5f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
@@ -6334,7 +6529,7 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6342,8 +6537,6 @@ define inreg <5 x double> @bitcast_v10i32_to_v5f64_scalar(<10 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6456,19 +6649,25 @@ define inreg <10 x i32> @bitcast_v5f64_to_v10i32_scalar(<5 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6485,19 +6684,25 @@ define inreg <10 x i32> @bitcast_v5f64_to_v10i32_scalar(<5 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6514,19 +6719,25 @@ define inreg <10 x i32> @bitcast_v5f64_to_v10i32_scalar(<5 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6548,19 +6759,22 @@ define inreg <10 x i32> @bitcast_v5f64_to_v10i32_scalar(<5 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -6700,10 +6914,18 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s23, s23, 3
@@ -6714,7 +6936,7 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6726,17 +6948,23 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v10i32_to_v5i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s23, s23, 3
@@ -6747,7 +6975,7 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6759,17 +6987,23 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v10i32_to_v5i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
; GFX9-NEXT: s_add_i32 s23, s23, 3
@@ -6780,7 +7014,7 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6792,19 +7026,22 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_v10i32_to_v5i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
; GFX11-NEXT: s_add_i32 s19, s19, 3
@@ -6815,7 +7052,7 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6823,8 +7060,6 @@ define inreg <5 x i64> @bitcast_v10i32_to_v5i64_scalar(<10 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6960,10 +7195,18 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s22, s22, 3
@@ -6974,7 +7217,7 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6986,17 +7229,23 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v5i64_to_v10i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s24, s24, 3
; VI-NEXT: s_addc_u32 s25, s25, 0
; VI-NEXT: s_add_u32 s22, s22, 3
@@ -7007,7 +7256,7 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7019,17 +7268,23 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v5i64_to_v10i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s24, s24, 3
; GFX9-NEXT: s_addc_u32 s25, s25, 0
; GFX9-NEXT: s_add_u32 s22, s22, 3
@@ -7040,7 +7295,7 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7052,19 +7307,22 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v5i64_to_v10i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s20, s20, 3
; GFX11-NEXT: s_addc_u32 s21, s21, 0
; GFX11-NEXT: s_add_u32 s18, s18, 3
@@ -7075,7 +7333,7 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7083,8 +7341,6 @@ define inreg <10 x i32> @bitcast_v5i64_to_v10i32_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7274,7 +7530,7 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s25, 16
; SI-NEXT: s_lshr_b32 s29, s23, 16
@@ -7286,8 +7542,26 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -7308,20 +7582,8 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -7342,7 +7604,7 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v12, s8
; SI-NEXT: v_mov_b32_e32 v11, s6
; SI-NEXT: v_mov_b32_e32 v10, s4
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
@@ -7379,10 +7641,18 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -7394,9 +7664,7 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7419,10 +7687,18 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -7434,9 +7710,7 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7464,11 +7738,16 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
@@ -7480,8 +7759,6 @@ define inreg <20 x i16> @bitcast_v10f32_to_v20i16_scalar(<10 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -7764,7 +8041,7 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_lshr_b32 s46, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -7796,8 +8073,17 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; SI-NEXT: s_and_b32 s13, s25, 0xffff
; SI-NEXT: s_lshl_b32 s14, s27, 16
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -7848,7 +8134,7 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -7860,18 +8146,23 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v20i16_to_v10f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s25, 3
; VI-NEXT: s_and_b32 s4, s25, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -7922,7 +8213,7 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7934,17 +8225,23 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v20i16_to_v10f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
@@ -7956,9 +8253,7 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7980,11 +8275,16 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
@@ -7996,8 +8296,6 @@ define inreg <10 x float> @bitcast_v20i16_to_v10f32_scalar(<20 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8194,7 +8492,7 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s25, 16
; SI-NEXT: s_lshr_b32 s29, s23, 16
@@ -8206,8 +8504,26 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -8228,20 +8544,8 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8262,7 +8566,7 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v12, s8
; SI-NEXT: v_mov_b32_e32 v11, s6
; SI-NEXT: v_mov_b32_e32 v10, s4
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
@@ -8299,10 +8603,18 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -8314,9 +8626,7 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8339,10 +8649,18 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -8354,9 +8672,7 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8384,11 +8700,16 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
@@ -8400,8 +8721,6 @@ define inreg <20 x half> @bitcast_v10f32_to_v20f16_scalar(<10 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8724,7 +9043,7 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_lshr_b32 s46, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_3
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s46, 16
@@ -8756,8 +9075,17 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; SI-NEXT: s_and_b32 s13, s25, 0xffff
; SI-NEXT: s_lshl_b32 s14, s27, 16
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB31_4
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s46
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s45
@@ -8839,10 +9167,7 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB31_2
-; SI-NEXT: .LBB31_4:
+; SI-NEXT: .LBB31_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -8859,10 +9184,18 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_3
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_4
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s25, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -8915,9 +9248,7 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_3:
-; VI-NEXT: s_branch .LBB31_2
-; VI-NEXT: .LBB31_4:
+; VI-NEXT: .LBB31_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8934,10 +9265,18 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
@@ -8950,9 +9289,7 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8974,11 +9311,16 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
@@ -8990,8 +9332,6 @@ define inreg <10 x float> @bitcast_v20f16_to_v10f32_scalar(<20 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -9883,7 +10223,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s91, s25, 24
; SI-NEXT: s_lshr_b32 s93, s25, 16
@@ -9915,60 +10255,10 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v13, s16, 1.0
-; SI-NEXT: v_add_f32_e64 v7, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v10, s18, 1.0
-; SI-NEXT: v_readfirstlane_b32 s16, v6
-; SI-NEXT: v_readfirstlane_b32 s17, v5
-; SI-NEXT: v_add_f32_e64 v3, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s24, 1.0
-; SI-NEXT: s_lshr_b64 s[26:27], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[28:29], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 8
-; SI-NEXT: v_readfirstlane_b32 s16, v10
-; SI-NEXT: v_readfirstlane_b32 s17, v7
-; SI-NEXT: v_readfirstlane_b32 s8, v2
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: v_readfirstlane_b32 s14, v4
-; SI-NEXT: v_readfirstlane_b32 s15, v3
-; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[46:47], s[16:17], 8
-; SI-NEXT: v_readfirstlane_b32 s16, v13
-; SI-NEXT: v_readfirstlane_b32 s17, v12
-; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 24
-; SI-NEXT: s_lshr_b64 s[6:7], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 8
-; SI-NEXT: s_lshr_b64 s[10:11], s[14:15], 24
-; SI-NEXT: s_lshr_b64 s[12:13], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 8
-; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v1
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v11, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; SI-NEXT: v_lshrrev_b32_e32 v17, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v5
-; SI-NEXT: v_lshrrev_b32_e32 v20, 24, v7
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v22, 8, v7
-; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v12
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v25, 8, v12
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
; SI-NEXT: ; implicit-def: $sgpr60
; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr56
@@ -9999,8 +10289,64 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr91
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v6, s20, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v13, s16, 1.0
+; SI-NEXT: v_add_f32_e64 v7, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v10, s18, 1.0
+; SI-NEXT: v_readfirstlane_b32 s16, v6
+; SI-NEXT: v_readfirstlane_b32 s17, v5
+; SI-NEXT: v_add_f32_e64 v3, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s22, 1.0
+; SI-NEXT: v_add_f32_e64 v1, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s24, 1.0
+; SI-NEXT: s_lshr_b64 s[26:27], s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[28:29], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 8
+; SI-NEXT: v_readfirstlane_b32 s16, v10
+; SI-NEXT: v_readfirstlane_b32 s17, v7
+; SI-NEXT: v_readfirstlane_b32 s8, v2
+; SI-NEXT: v_readfirstlane_b32 s9, v1
+; SI-NEXT: v_readfirstlane_b32 s14, v4
+; SI-NEXT: v_readfirstlane_b32 s15, v3
+; SI-NEXT: s_lshr_b64 s[42:43], s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[46:47], s[16:17], 8
+; SI-NEXT: v_readfirstlane_b32 s16, v13
+; SI-NEXT: v_readfirstlane_b32 s17, v12
+; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 24
+; SI-NEXT: s_lshr_b64 s[6:7], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 8
+; SI-NEXT: s_lshr_b64 s[10:11], s[14:15], 24
+; SI-NEXT: s_lshr_b64 s[12:13], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[14:15], s[14:15], 8
+; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v11, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; SI-NEXT: v_lshrrev_b32_e32 v17, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v19, 8, v5
+; SI-NEXT: v_lshrrev_b32_e32 v20, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v22, 8, v7
+; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v12
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v25, 8, v12
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v13, s16
; SI-NEXT: v_mov_b32_e32 v12, s17
; SI-NEXT: v_mov_b32_e32 v10, s18
@@ -10026,7 +10372,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v11, s94
; SI-NEXT: v_mov_b32_e32 v9, s93
; SI-NEXT: v_mov_b32_e32 v8, s91
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_and_b32_e32 v13, 0xff, v13
; SI-NEXT: s_lshl_b32 s5, s60, 8
; SI-NEXT: v_or_b32_e32 v13, s5, v13
@@ -10152,7 +10498,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s26, s25, 24
; VI-NEXT: s_lshr_b32 s27, s25, 16
@@ -10184,8 +10530,46 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s25, 1.0
; VI-NEXT: v_add_f32_e64 v1, s24, 1.0
; VI-NEXT: v_add_f32_e64 v4, s23, 1.0
@@ -10226,40 +10610,8 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; VI-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v9, s16
; VI-NEXT: v_mov_b32_e32 v10, s17
; VI-NEXT: v_mov_b32_e32 v7, s18
@@ -10300,7 +10652,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v13, s8
; VI-NEXT: v_mov_b32_e32 v12, s10
; VI-NEXT: v_mov_b32_e32 v11, s12
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v15, v48, v15, s4
; VI-NEXT: v_perm_b32 v9, v9, v39, s4
@@ -10368,7 +10720,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s26, s25, 24
; GFX9-NEXT: s_lshr_b32 s27, s25, 16
@@ -10400,8 +10752,46 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v4, s23, 1.0
@@ -10442,40 +10832,8 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v9, s16
; GFX9-NEXT: v_mov_b32_e32 v10, s17
; GFX9-NEXT: v_mov_b32_e32 v7, s18
@@ -10516,7 +10874,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v13, s8
; GFX9-NEXT: v_mov_b32_e32 v12, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s12
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v15, v48, v15, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v39, s4
@@ -10576,7 +10934,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s15, s21, 24
; GFX11-NEXT: s_lshr_b32 s22, s21, 16
@@ -10608,9 +10966,46 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB33_3
+; GFX11-NEXT: .LBB33_2:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: .LBB33_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v2, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v4, s19, 1.0
@@ -10651,40 +11046,8 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX11-NEXT: s_branch .LBB33_5
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: s_branch .LBB33_2
-; GFX11-NEXT: .LBB33_4:
+; GFX11-NEXT: s_branch .LBB33_6
+; GFX11-NEXT: .LBB33_5:
; GFX11-NEXT: v_dual_mov_b32 v9, s0 :: v_dual_mov_b32 v10, s1
; GFX11-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v8, s3
; GFX11-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v6, s17
@@ -10705,7 +11068,7 @@ define inreg <40 x i8> @bitcast_v10f32_to_v40i8_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v15, s4
; GFX11-NEXT: v_dual_mov_b32 v14, s6 :: v_dual_mov_b32 v13, s8
; GFX11-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v11, s12
-; GFX11-NEXT: .LBB33_5: ; %end
+; GFX11-NEXT: .LBB33_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v15, v48, v15, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v9, v9, v39, 0xc0c0004
@@ -11999,7 +12362,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s79, v2
; SI-NEXT: v_readfirstlane_b32 s88, v1
; SI-NEXT: v_readfirstlane_b32 s89, v0
-; SI-NEXT: s_cbranch_scc0 .LBB35_4
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -12091,8 +12454,17 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_cbranch_execnz .LBB35_3
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s14, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s14, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -12223,7 +12595,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
-; SI-NEXT: .LBB35_3: ; %end
+; SI-NEXT: .LBB35_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -12235,9 +12607,6 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s12
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13
-; SI-NEXT: s_branch .LBB35_2
;
; VI-LABEL: bitcast_v40i8_to_v10f32_scalar:
; VI: ; %bb.0:
@@ -12270,7 +12639,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s62, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s63, v0
-; VI-NEXT: s_cbranch_scc0 .LBB35_4
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -12333,8 +12702,17 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; VI-NEXT: v_perm_b32 v9, s7, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB35_3
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -12436,11 +12814,8 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB35_3: ; %end
+; VI-NEXT: .LBB35_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; VI-NEXT: s_branch .LBB35_2
;
; GFX9-LABEL: bitcast_v40i8_to_v10f32_scalar:
; GFX9: ; %bb.0:
@@ -12473,7 +12848,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s62, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -12536,8 +12911,17 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v9, s7, v11, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: v_or_b32_e32 v9, v10, v9
-; GFX9-NEXT: s_cbranch_execnz .LBB35_3
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -12630,11 +13014,8 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v10
; GFX9-NEXT: v_add_u32_sdwa v9, v9, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB35_3: ; %end
+; GFX9-NEXT: .LBB35_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX9-NEXT: s_branch .LBB35_2
;
; GFX11-LABEL: bitcast_v40i8_to_v10f32_scalar:
; GFX11: ; %bb.0:
@@ -12664,7 +13045,7 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s57, v0
; GFX11-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-NEXT: s_mov_b32 s58, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB35_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12708,9 +13089,17 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v7, v12, v10
; GFX11-NEXT: v_or_b32_e32 v8, v14, v11
; GFX11-NEXT: v_or_b32_e32 v9, v15, v13
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_3
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB35_3
+; GFX11-NEXT: .LBB35_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-NEXT: .LBB35_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -12802,11 +13191,8 @@ define inreg <10 x float> @bitcast_v40i8_to_v10f32_scalar(<40 x i8> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_or_b32_e32 v8, v12, v13
; GFX11-NEXT: v_or_b32_e32 v9, v14, v15
-; GFX11-NEXT: .LBB35_3: ; %end
+; GFX11-NEXT: .LBB35_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX11-NEXT: s_branch .LBB35_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -12933,10 +13319,18 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -12948,9 +13342,7 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -12973,10 +13365,18 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -12988,9 +13388,7 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13013,10 +13411,18 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -13028,9 +13434,7 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13058,11 +13462,16 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
@@ -13074,8 +13483,6 @@ define inreg <5 x double> @bitcast_v10f32_to_v5f64_scalar(<10 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -13198,19 +13605,25 @@ define inreg <10 x float> @bitcast_v5f64_to_v10f32_scalar(<5 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13227,19 +13640,25 @@ define inreg <10 x float> @bitcast_v5f64_to_v10f32_scalar(<5 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13256,19 +13675,25 @@ define inreg <10 x float> @bitcast_v5f64_to_v10f32_scalar(<5 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13290,19 +13715,22 @@ define inreg <10 x float> @bitcast_v5f64_to_v10f32_scalar(<5 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -13436,10 +13864,18 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
; SI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -13451,9 +13887,7 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13476,10 +13910,18 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
; VI-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -13491,9 +13933,7 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13516,10 +13956,18 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
; GFX9-NEXT: v_add_f32_e64 v7, s23, 1.0
@@ -13531,9 +13979,7 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13561,11 +14007,16 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
; GFX11-NEXT: v_add_f32_e64 v7, s19, 1.0
@@ -13577,8 +14028,6 @@ define inreg <5 x i64> @bitcast_v10f32_to_v5i64_scalar(<10 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -13724,10 +14173,18 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s22, s22, 3
@@ -13738,7 +14195,7 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13750,17 +14207,23 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v5i64_to_v10f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s24, s24, 3
; VI-NEXT: s_addc_u32 s25, s25, 0
; VI-NEXT: s_add_u32 s22, s22, 3
@@ -13771,7 +14234,7 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13783,17 +14246,23 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v5i64_to_v10f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_3
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s24, s24, 3
; GFX9-NEXT: s_addc_u32 s25, s25, 0
; GFX9-NEXT: s_add_u32 s22, s22, 3
@@ -13804,7 +14273,7 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB43_3: ; %end
+; GFX9-NEXT: .LBB43_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13816,19 +14285,22 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_4:
-; GFX9-NEXT: s_branch .LBB43_2
;
; GFX11-LABEL: bitcast_v5i64_to_v10f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_3
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s20, s20, 3
; GFX11-NEXT: s_addc_u32 s21, s21, 0
; GFX11-NEXT: s_add_u32 s18, s18, 3
@@ -13839,7 +14311,7 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB43_3: ; %end
+; GFX11-NEXT: .LBB43_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13847,8 +14319,6 @@ define inreg <10 x float> @bitcast_v5i64_to_v10f32_scalar(<5 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_4:
-; GFX11-NEXT: s_branch .LBB43_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14166,7 +14636,7 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s46, s17, 16
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s46, 16
@@ -14207,9 +14677,27 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_mov_b32 s9, s43
; SI-NEXT: s_lshr_b64 s[42:43], s[72:73], 16
; SI-NEXT: s_mov_b32 s7, s45
+; SI-NEXT: s_mov_b64 s[44:45], 0
; SI-NEXT: s_mov_b32 s5, s73
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[44:45], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -14270,7 +14758,7 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s56, s9, 16
; SI-NEXT: s_lshr_b32 s57, s7, 16
; SI-NEXT: s_lshr_b32 s58, s5, 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -14312,27 +14800,23 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v20i16_to_v20f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_add_i32 s7, s17, 3
@@ -14383,7 +14867,7 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14395,17 +14879,23 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v20i16_to_v20f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
@@ -14417,9 +14907,7 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14447,11 +14935,16 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
@@ -14463,8 +14956,6 @@ define inreg <20 x half> @bitcast_v20i16_to_v20f16_scalar(<20 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -14754,10 +15245,18 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
@@ -14843,10 +15342,8 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[14:15], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[10:11], v[8:9], 16
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v21, s11
; SI-NEXT: v_mov_b32_e32 v22, s9
; SI-NEXT: v_mov_b32_e32 v23, s8
@@ -14867,7 +15364,7 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v14, s13
; SI-NEXT: v_mov_b32_e32 v12, s12
; SI-NEXT: v_mov_b32_e32 v10, s10
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v20
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -14904,10 +15401,18 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s24, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s5
@@ -14960,9 +15465,7 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v0, v1
; VI-NEXT: v_or_b32_e32 v0, v10, v11
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14985,10 +15488,18 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
@@ -15001,9 +15512,7 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15031,11 +15540,16 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
@@ -15047,8 +15561,6 @@ define inreg <20 x i16> @bitcast_v20f16_to_v20i16_scalar(<20 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -16119,22 +16631,20 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v3, s34, 0
-; SI-NEXT: v_writelane_b32 v3, s35, 1
-; SI-NEXT: v_writelane_b32 v3, s30, 2
-; SI-NEXT: v_writelane_b32 v3, s31, 3
+; SI-NEXT: v_writelane_b32 v3, s30, 0
+; SI-NEXT: v_writelane_b32 v3, s31, 1
; SI-NEXT: s_lshr_b32 s90, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
; SI-NEXT: s_lshr_b32 s91, s23, 16
-; SI-NEXT: s_lshr_b32 s34, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s22, 16
; SI-NEXT: s_lshr_b32 s92, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s20, 16
; SI-NEXT: s_lshr_b32 s93, s19, 16
-; SI-NEXT: s_lshr_b32 s30, s18, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s18, 16
; SI-NEXT: s_lshr_b32 s94, s17, 16
; SI-NEXT: s_lshr_b32 s95, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_4
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -16143,26 +16653,26 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s13, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s10, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_or_b32 s11, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s8, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
; SI-NEXT: s_lshl_b32 s5, s92, 16
; SI-NEXT: s_or_b32 s9, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s6, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[12:13], 24
; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s25, 0xffff
; SI-NEXT: s_lshl_b32 s15, s90, 16
@@ -16191,11 +16701,49 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 24
; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 8
-; SI-NEXT: s_cbranch_execnz .LBB49_3
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[88:89], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[88:89], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[88:89], s[88:89], exec
+; SI-NEXT: s_cselect_b32 s63, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s63, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s25, 0xffff
@@ -16203,7 +16751,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
+; SI-NEXT: s_lshl_b32 s7, s30, 16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s23, 0xffff
@@ -16211,7 +16759,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s31, 16
+; SI-NEXT: s_lshl_b32 s9, vcc_hi, 16
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s21, 0xffff
@@ -16219,7 +16767,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s30, 16
+; SI-NEXT: s_lshl_b32 s11, vcc_lo, 16
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s19, 0xffff
@@ -16273,7 +16821,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s15, s5, 24
; SI-NEXT: s_lshr_b32 s90, s5, 16
; SI-NEXT: s_lshr_b32 s27, s5, 8
-; SI-NEXT: .LBB49_3: ; %end
+; SI-NEXT: .LBB49_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: s_lshl_b32 s16, s44, 8
; SI-NEXT: s_or_b32 s12, s12, s16
@@ -16400,54 +16948,20 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
; SI-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s30, v3, 2
+; SI-NEXT: v_readlane_b32 s30, v3, 0
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT: v_readlane_b32 s31, v3, 3
-; SI-NEXT: v_readlane_b32 s35, v3, 1
-; SI-NEXT: v_readlane_b32 s34, v3, 0
+; SI-NEXT: v_readlane_b32 s31, v3, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB49_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: s_branch .LBB49_2
;
; VI-LABEL: bitcast_v20i16_to_v40i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_4
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s26, s25, 24
; VI-NEXT: s_lshr_b32 s27, s25, 16
@@ -16479,8 +16993,46 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB49_3
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -16561,7 +17113,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b32 s74, s17, 8
; VI-NEXT: s_lshr_b32 s75, s16, 16
; VI-NEXT: s_lshr_b32 s76, s16, 8
-; VI-NEXT: .LBB49_3: ; %end
+; VI-NEXT: .LBB49_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: v_mov_b32_e32 v1, s76
@@ -16644,44 +17196,12 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_4:
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: s_branch .LBB49_2
;
; GFX9-LABEL: bitcast_v20i16_to_v40i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s26, s25, 24
; GFX9-NEXT: s_lshr_b32 s27, s25, 16
@@ -16713,8 +17233,46 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v2, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v4, s23, 3 op_sel_hi:[1,0]
@@ -16755,40 +17313,8 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v9, s16
; GFX9-NEXT: v_mov_b32_e32 v10, s17
; GFX9-NEXT: v_mov_b32_e32 v7, s18
@@ -16829,7 +17355,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s8
; GFX9-NEXT: v_mov_b32_e32 v14, s6
; GFX9-NEXT: v_mov_b32_e32 v15, s4
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v15, v48, v15, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v39, s4
@@ -16889,7 +17415,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s15, s21, 24
; GFX11-NEXT: s_lshr_b32 s22, s21, 16
@@ -16921,9 +17447,46 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB49_3
+; GFX11-NEXT: .LBB49_2:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: .LBB49_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v2, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v4, s19, 3 op_sel_hi:[1,0]
@@ -16964,40 +17527,8 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: s_branch .LBB49_2
-; GFX11-NEXT: .LBB49_4:
+; GFX11-NEXT: s_branch .LBB49_6
+; GFX11-NEXT: .LBB49_5:
; GFX11-NEXT: v_dual_mov_b32 v9, s0 :: v_dual_mov_b32 v10, s1
; GFX11-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v8, s3
; GFX11-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v6, s17
@@ -17018,7 +17549,7 @@ define inreg <40 x i8> @bitcast_v20i16_to_v40i8_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v11, s12
; GFX11-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s8
; GFX11-NEXT: v_dual_mov_b32 v14, s6 :: v_dual_mov_b32 v15, s4
-; GFX11-NEXT: .LBB49_5: ; %end
+; GFX11-NEXT: .LBB49_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v15, v48, v15, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v9, v9, v39, 0xc0c0004
@@ -18439,10 +18970,10 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s91, v24
; SI-NEXT: v_readfirstlane_b32 s93, v23
; SI-NEXT: v_readfirstlane_b32 s95, v22
-; SI-NEXT: v_readfirstlane_b32 s30, v21
-; SI-NEXT: v_readfirstlane_b32 s31, v20
-; SI-NEXT: v_readfirstlane_b32 s34, v19
-; SI-NEXT: v_readfirstlane_b32 s35, v18
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v21
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v20
+; SI-NEXT: v_readfirstlane_b32 s30, v19
+; SI-NEXT: v_readfirstlane_b32 s31, v18
; SI-NEXT: v_readfirstlane_b32 s74, v17
; SI-NEXT: v_readfirstlane_b32 s75, v16
; SI-NEXT: v_readfirstlane_b32 s77, v15
@@ -18462,7 +18993,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s58, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s59, v0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -18492,8 +19023,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s88, 24
; SI-NEXT: s_or_b32 s44, s7, s5
-; SI-NEXT: s_and_b32 s5, s35, 0xff
-; SI-NEXT: s_lshl_b32 s7, s34, 8
+; SI-NEXT: s_and_b32 s5, s31, 0xff
+; SI-NEXT: s_lshl_b32 s7, s30, 8
; SI-NEXT: s_or_b32 s12, s5, s7
; SI-NEXT: s_and_b32 s5, s20, 0xff
; SI-NEXT: s_lshl_b32 s7, s21, 8
@@ -18516,8 +19047,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s60, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_or_b32 vcc_lo, s15, s14
-; SI-NEXT: s_or_b32 s43, s13, vcc_lo
+; SI-NEXT: s_or_b32 s34, s15, s14
+; SI-NEXT: s_or_b32 s43, s13, s34
; SI-NEXT: s_and_b32 s13, s79, 0xff
; SI-NEXT: s_lshl_b32 s14, s77, 8
; SI-NEXT: s_or_b32 s13, s13, s14
@@ -18525,8 +19056,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s74, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_or_b32 vcc_hi, s15, s14
-; SI-NEXT: s_or_b32 s45, s13, vcc_hi
+; SI-NEXT: s_or_b32 s35, s15, s14
+; SI-NEXT: s_or_b32 s45, s13, s35
; SI-NEXT: s_and_b32 s13, s95, 0xff
; SI-NEXT: s_lshl_b32 s14, s93, 8
; SI-NEXT: s_or_b32 s13, s13, s14
@@ -18534,11 +19065,11 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s90, 24
; SI-NEXT: s_or_b32 s36, s15, s14
-; SI-NEXT: s_and_b32 s14, s31, 0xff
+; SI-NEXT: s_and_b32 s14, vcc_hi, 0xff
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s30, 24
+; SI-NEXT: s_lshl_b32 s15, vcc_lo, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s56
; SI-NEXT: s_or_b32 s7, s7, s57
@@ -18566,19 +19097,42 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
; SI-NEXT: s_lshr_b32 s41, s56, 16
; SI-NEXT: s_lshr_b32 s43, s57, 16
-; SI-NEXT: s_lshr_b32 s45, vcc_lo, 16
-; SI-NEXT: s_lshr_b32 s47, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s45, s34, 16
+; SI-NEXT: s_lshr_b32 s47, s35, 16
; SI-NEXT: s_lshr_b32 s15, s36, 16
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s35, s35, 3
-; SI-NEXT: s_and_b32 s4, s35, 0xff
-; SI-NEXT: s_lshl_b32 s5, s34, 8
+; SI-NEXT: s_mov_b64 s[56:57], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[56:57], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[56:57], s[56:57], exec
+; SI-NEXT: s_cselect_b32 s56, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s56, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_and_b32 s4, s31, 0xff
+; SI-NEXT: s_lshl_b32 s5, s30, 8
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s31, 0xff
+; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s30, 24
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -18711,7 +19265,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s45, s9, 16
; SI-NEXT: s_lshr_b32 s47, s7, 16
; SI-NEXT: s_lshr_b32 s15, s5, 16
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -18765,23 +19319,6 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v40i8_to_v20i16_scalar:
; VI: ; %bb.0:
@@ -18814,7 +19351,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s10, v0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -18877,8 +19414,17 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v9, s63, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s63, s63, 3
; VI-NEXT: v_mov_b32_e32 v0, s62
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -18980,11 +19526,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v40i8_to_v20i16_scalar:
; GFX9: ; %bb.0:
@@ -19017,7 +19560,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s41, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
@@ -19083,8 +19626,17 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
; GFX9-NEXT: v_mov_b32_e32 v8, s4
-; GFX9-NEXT: s_cbranch_execnz .LBB51_3
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s63, s63, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s62
; GFX9-NEXT: v_mov_b32_e32 v6, 0xc0c0004
@@ -19186,11 +19738,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
-; GFX9-NEXT: .LBB51_3: ; %end
+; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB51_2
;
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20i16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -19220,7 +19769,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s59, s43, 0xff
@@ -19261,9 +19810,17 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s15, s5, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s59
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-TRUE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB51_3
+; GFX11-TRUE16-NEXT: .LBB51_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s58, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
@@ -19335,11 +19892,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: .LBB51_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB51_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB51_2
;
; GFX11-FAKE16-LABEL: bitcast_v40i8_to_v20i16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -19369,7 +19923,7 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s58, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s59, s56, 0xff
@@ -19418,9 +19972,17 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s59
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v7, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v13, 16, v11
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-FAKE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB51_3
+; GFX11-FAKE16-NEXT: .LBB51_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s58, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s57, s57, 3
; GFX11-FAKE16-NEXT: s_add_i32 s46, s46, 3
@@ -19502,11 +20064,8 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v11, 16, v7
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v8, v8, 16, v15
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v9, 16, v16
-; GFX11-FAKE16-NEXT: .LBB51_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB51_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB51_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -19791,7 +20350,7 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -19823,8 +20382,17 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -19875,7 +20443,7 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s45, s4, 0x30000
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -19899,18 +20467,23 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v20i16_to_v5f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s25, 3
; VI-NEXT: s_and_b32 s4, s25, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -19961,7 +20534,7 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -19973,17 +20546,23 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v20i16_to_v5f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
@@ -19995,9 +20574,7 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20025,11 +20602,16 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
@@ -20041,8 +20623,6 @@ define inreg <5 x double> @bitcast_v20i16_to_v5f64_scalar(<20 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_3:
-; GFX11-NEXT: s_branch .LBB53_2
; GFX11-NEXT: .LBB53_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -20228,7 +20808,7 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s25, 16
; SI-NEXT: s_lshr_b32 s29, s23, 16
@@ -20240,8 +20820,26 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -20257,20 +20855,8 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: v_mov_b32_e32 v5, s21
@@ -20291,7 +20877,7 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s8
; SI-NEXT: v_mov_b32_e32 v11, s6
; SI-NEXT: v_mov_b32_e32 v10, s4
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
@@ -20328,19 +20914,25 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -20363,19 +20955,25 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20403,19 +21001,22 @@ define inreg <20 x i16> @bitcast_v5f64_to_v20i16_scalar(<5 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -20710,7 +21311,7 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -20742,8 +21343,17 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -20794,7 +21404,7 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s45, s4, 0x30000
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -20818,18 +21428,23 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v20i16_to_v5i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s25, 3
; VI-NEXT: s_and_b32 s4, s25, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -20880,7 +21495,7 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -20892,17 +21507,23 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v20i16_to_v5i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v7, s23, 3 op_sel_hi:[1,0]
@@ -20914,9 +21535,7 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20944,11 +21563,16 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v7, s19, 3 op_sel_hi:[1,0]
@@ -20960,8 +21584,6 @@ define inreg <5 x i64> @bitcast_v20i16_to_v5i64_scalar(<20 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: s_branch .LBB57_2
; GFX11-NEXT: .LBB57_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -21170,7 +21792,7 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s26, s25, 16
; SI-NEXT: s_lshr_b32 s27, s23, 16
@@ -21182,8 +21804,26 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s22, s22, 3
@@ -21204,7 +21844,7 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -21246,27 +21886,23 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v5i64_to_v20i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s24, s24, 3
; VI-NEXT: s_addc_u32 s25, s25, 0
; VI-NEXT: s_add_u32 s22, s22, 3
@@ -21277,7 +21913,7 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -21289,17 +21925,23 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v5i64_to_v20i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_3
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s24, s24, 3
; GFX9-NEXT: s_addc_u32 s25, s25, 0
; GFX9-NEXT: s_add_u32 s22, s22, 3
@@ -21310,7 +21952,7 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB59_3: ; %end
+; GFX9-NEXT: .LBB59_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -21322,19 +21964,22 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_4:
-; GFX9-NEXT: s_branch .LBB59_2
;
; GFX11-LABEL: bitcast_v5i64_to_v20i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_3
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s20, s20, 3
; GFX11-NEXT: s_addc_u32 s21, s21, 0
; GFX11-NEXT: s_add_u32 s18, s18, 3
@@ -21345,7 +21990,7 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB59_3: ; %end
+; GFX11-NEXT: .LBB59_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -21353,8 +21998,6 @@ define inreg <20 x i16> @bitcast_v5i64_to_v20i16_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_4:
-; GFX11-NEXT: s_branch .LBB59_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -22399,14 +23042,12 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v12, s34, 0
-; SI-NEXT: v_writelane_b32 v12, s35, 1
-; SI-NEXT: v_writelane_b32 v12, s30, 2
-; SI-NEXT: v_writelane_b32 v12, s31, 3
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
+; SI-NEXT: v_writelane_b32 v12, s30, 0
+; SI-NEXT: v_writelane_b32 v12, s31, 1
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
; SI-NEXT: s_lshr_b32 s92, s19, 16
@@ -22414,7 +23055,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s90, s17, 16
; SI-NEXT: s_lshr_b32 s91, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB61_3
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -22435,17 +23076,17 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s9, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s6, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[12:13], 24
; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s34, 16
+; SI-NEXT: s_lshl_b32 s15, s30, 16
; SI-NEXT: s_lshr_b64 s[26:27], s[12:13], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[12:13], 8
; SI-NEXT: s_lshr_b64 s[28:29], s[10:11], 24
@@ -22464,18 +23105,56 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_bfe_u32 s45, s90, 0x80008
; SI-NEXT: s_bfe_u32 s47, s92, 0x80008
; SI-NEXT: s_bfe_u32 s57, s94, 0x80008
-; SI-NEXT: s_bfe_u32 s59, s30, 0x80008
-; SI-NEXT: s_bfe_u32 s61, s34, 0x80008
+; SI-NEXT: s_bfe_u32 s59, vcc_lo, 0x80008
+; SI-NEXT: s_bfe_u32 s61, s30, 0x80008
; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 8
; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 24
; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 8
-; SI-NEXT: s_cbranch_execnz .LBB61_4
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[88:89], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[88:89], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[88:89], s[88:89], exec
+; SI-NEXT: s_cselect_b32 s63, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s63, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v2, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s30
; SI-NEXT: v_cvt_f32_f16_e32 v7, s16
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -22484,7 +23163,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v3
; SI-NEXT: v_readfirstlane_b32 s5, v2
; SI-NEXT: v_cvt_f32_f16_e32 v2, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v3, vcc_hi
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
@@ -22499,7 +23178,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v3, vcc_lo
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v2
; SI-NEXT: s_lshl_b32 s6, s6, 16
@@ -22598,42 +23277,10 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_bfe_u32 v8, v3, 8, 8
; SI-NEXT: v_bfe_u32 v6, v2, 8, 8
; SI-NEXT: v_bfe_u32 v5, v1, 8, 8
-; SI-NEXT: s_branch .LBB61_5
-; SI-NEXT: .LBB61_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: s_branch .LBB61_2
-; SI-NEXT: .LBB61_4:
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_mov_b32_e32 v2, s30
+; SI-NEXT: s_branch .LBB61_6
+; SI-NEXT: .LBB61_5:
+; SI-NEXT: v_mov_b32_e32 v1, s30
+; SI-NEXT: v_mov_b32_e32 v2, vcc_lo
; SI-NEXT: v_mov_b32_e32 v3, s94
; SI-NEXT: v_mov_b32_e32 v4, s92
; SI-NEXT: v_mov_b32_e32 v7, s90
@@ -22642,7 +23289,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s57
; SI-NEXT: v_mov_b32_e32 v9, s47
; SI-NEXT: v_mov_b32_e32 v10, s45
-; SI-NEXT: .LBB61_5: ; %end
+; SI-NEXT: .LBB61_6: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: s_lshl_b32 s16, s40, 8
; SI-NEXT: s_or_b32 s12, s12, s16
@@ -22760,11 +23407,9 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 36, v0
-; SI-NEXT: v_readlane_b32 s30, v12, 2
+; SI-NEXT: v_readlane_b32 s30, v12, 0
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT: v_readlane_b32 s31, v12, 3
-; SI-NEXT: v_readlane_b32 s35, v12, 1
-; SI-NEXT: v_readlane_b32 s34, v12, 0
+; SI-NEXT: v_readlane_b32 s31, v12, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -22775,7 +23420,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB61_3
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s25, 24
; VI-NEXT: s_lshr_b32 s59, s25, 16
@@ -22807,8 +23452,46 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB61_4
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_add_f16_e32 v7, s4, v1
@@ -22880,40 +23563,8 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; VI-NEXT: v_bfe_u32 v32, v9, 8, 8
; VI-NEXT: v_bfe_u32 v35, v8, 8, 8
; VI-NEXT: v_bfe_u32 v38, v7, 8, 8
-; VI-NEXT: s_branch .LBB61_5
-; VI-NEXT: .LBB61_3:
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB61_2
-; VI-NEXT: .LBB61_4:
+; VI-NEXT: s_branch .LBB61_6
+; VI-NEXT: .LBB61_5:
; VI-NEXT: v_mov_b32_e32 v12, s76
; VI-NEXT: v_mov_b32_e32 v7, s75
; VI-NEXT: v_mov_b32_e32 v13, s74
@@ -22954,7 +23605,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v3, s8
; VI-NEXT: v_mov_b32_e32 v4, s6
; VI-NEXT: v_mov_b32_e32 v5, s4
-; VI-NEXT: .LBB61_5: ; %end
+; VI-NEXT: .LBB61_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v5, v12, v5, s4
; VI-NEXT: v_perm_b32 v12, v21, v48, s4
@@ -23022,7 +23673,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s26, s25, 24
; GFX9-NEXT: s_lshr_b32 s27, s25, 16
@@ -23054,8 +23705,46 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB61_4
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v1, 0x200
; GFX9-NEXT: v_pk_add_f16 v10, s17, v1 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v9, s16, v1 op_sel_hi:[1,0]
@@ -23097,40 +23786,8 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX9-NEXT: s_branch .LBB61_5
-; GFX9-NEXT: .LBB61_3:
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: s_branch .LBB61_2
-; GFX9-NEXT: .LBB61_4:
+; GFX9-NEXT: s_branch .LBB61_6
+; GFX9-NEXT: .LBB61_5:
; GFX9-NEXT: v_mov_b32_e32 v9, s16
; GFX9-NEXT: v_mov_b32_e32 v10, s17
; GFX9-NEXT: v_mov_b32_e32 v7, s18
@@ -23171,7 +23828,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s8
; GFX9-NEXT: v_mov_b32_e32 v14, s6
; GFX9-NEXT: v_mov_b32_e32 v15, s4
-; GFX9-NEXT: .LBB61_5: ; %end
+; GFX9-NEXT: .LBB61_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v15, v48, v15, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v39, s4
@@ -23231,7 +23888,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB61_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s15, s21, 24
; GFX11-NEXT: s_lshr_b32 s22, s21, 16
@@ -23263,9 +23920,46 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_4
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB61_3
+; GFX11-NEXT: .LBB61_2:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: .LBB61_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s19 op_sel_hi:[0,1]
@@ -23306,40 +24000,8 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v38, 8, v10
; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v39, 8, v9
-; GFX11-NEXT: s_branch .LBB61_5
-; GFX11-NEXT: .LBB61_3:
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: s_branch .LBB61_2
-; GFX11-NEXT: .LBB61_4:
+; GFX11-NEXT: s_branch .LBB61_6
+; GFX11-NEXT: .LBB61_5:
; GFX11-NEXT: v_dual_mov_b32 v9, s0 :: v_dual_mov_b32 v10, s1
; GFX11-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v8, s3
; GFX11-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v6, s17
@@ -23360,7 +24022,7 @@ define inreg <40 x i8> @bitcast_v20f16_to_v40i8_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v11, s12
; GFX11-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v13, s8
; GFX11-NEXT: v_dual_mov_b32 v14, s6 :: v_dual_mov_b32 v15, s4
-; GFX11-NEXT: .LBB61_5: ; %end
+; GFX11-NEXT: .LBB61_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v15, v48, v15, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v9, v9, v39, 0xc0c0004
@@ -24781,10 +25443,10 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s91, v24
; SI-NEXT: v_readfirstlane_b32 s93, v23
; SI-NEXT: v_readfirstlane_b32 s95, v22
-; SI-NEXT: v_readfirstlane_b32 s30, v21
-; SI-NEXT: v_readfirstlane_b32 s31, v20
-; SI-NEXT: v_readfirstlane_b32 s34, v19
-; SI-NEXT: v_readfirstlane_b32 s35, v18
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v21
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v20
+; SI-NEXT: v_readfirstlane_b32 s30, v19
+; SI-NEXT: v_readfirstlane_b32 s31, v18
; SI-NEXT: v_readfirstlane_b32 s74, v17
; SI-NEXT: v_readfirstlane_b32 s75, v16
; SI-NEXT: v_readfirstlane_b32 s77, v15
@@ -24804,7 +25466,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s58, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s59, v0
-; SI-NEXT: s_cbranch_scc0 .LBB63_4
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -24834,8 +25496,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s88, 24
; SI-NEXT: s_or_b32 s44, s7, s5
-; SI-NEXT: s_and_b32 s5, s35, 0xff
-; SI-NEXT: s_lshl_b32 s7, s34, 8
+; SI-NEXT: s_and_b32 s5, s31, 0xff
+; SI-NEXT: s_lshl_b32 s7, s30, 8
; SI-NEXT: s_or_b32 s12, s5, s7
; SI-NEXT: s_and_b32 s5, s20, 0xff
; SI-NEXT: s_lshl_b32 s7, s21, 8
@@ -24858,8 +25520,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s60, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_or_b32 vcc_lo, s15, s14
-; SI-NEXT: s_or_b32 s43, s13, vcc_lo
+; SI-NEXT: s_or_b32 s34, s15, s14
+; SI-NEXT: s_or_b32 s43, s13, s34
; SI-NEXT: s_and_b32 s13, s79, 0xff
; SI-NEXT: s_lshl_b32 s14, s77, 8
; SI-NEXT: s_or_b32 s13, s13, s14
@@ -24867,8 +25529,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s74, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_or_b32 vcc_hi, s15, s14
-; SI-NEXT: s_or_b32 s45, s13, vcc_hi
+; SI-NEXT: s_or_b32 s35, s15, s14
+; SI-NEXT: s_or_b32 s45, s13, s35
; SI-NEXT: s_and_b32 s13, s95, 0xff
; SI-NEXT: s_lshl_b32 s14, s93, 8
; SI-NEXT: s_or_b32 s13, s13, s14
@@ -24876,11 +25538,11 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_lshl_b32 s15, s90, 24
; SI-NEXT: s_or_b32 s36, s15, s14
-; SI-NEXT: s_and_b32 s14, s31, 0xff
+; SI-NEXT: s_and_b32 s14, vcc_hi, 0xff
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s30, 24
+; SI-NEXT: s_lshl_b32 s15, vcc_lo, 24
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s56
; SI-NEXT: s_or_b32 s7, s7, s57
@@ -24908,19 +25570,42 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
; SI-NEXT: s_lshr_b32 s41, s56, 16
; SI-NEXT: s_lshr_b32 s43, s57, 16
-; SI-NEXT: s_lshr_b32 s45, vcc_lo, 16
-; SI-NEXT: s_lshr_b32 s47, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s45, s34, 16
+; SI-NEXT: s_lshr_b32 s47, s35, 16
; SI-NEXT: s_lshr_b32 s15, s36, 16
-; SI-NEXT: s_cbranch_execnz .LBB63_3
-; SI-NEXT: .LBB63_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s35, s35, 3
-; SI-NEXT: s_and_b32 s4, s35, 0xff
-; SI-NEXT: s_lshl_b32 s5, s34, 8
+; SI-NEXT: s_mov_b64 s[56:57], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[56:57], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[56:57], s[56:57], exec
+; SI-NEXT: s_cselect_b32 s56, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s56, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_and_b32 s4, s31, 0xff
+; SI-NEXT: s_lshl_b32 s5, s30, 8
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s31, 0xff
+; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s30, 24
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -25053,7 +25738,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s45, s9, 16
; SI-NEXT: s_lshr_b32 s47, s7, 16
; SI-NEXT: s_lshr_b32 s15, s5, 16
-; SI-NEXT: .LBB63_3: ; %end
+; SI-NEXT: .LBB63_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -25107,23 +25792,6 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB63_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: s_branch .LBB63_2
;
; VI-LABEL: bitcast_v40i8_to_v20f16_scalar:
; VI: ; %bb.0:
@@ -25156,7 +25824,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s10, v0
-; VI-NEXT: s_cbranch_scc0 .LBB63_4
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -25219,8 +25887,17 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v9, s63, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB63_3
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s63, s63, 3
; VI-NEXT: v_mov_b32_e32 v0, s62
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -25322,11 +25999,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB63_3: ; %end
+; VI-NEXT: .LBB63_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB63_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB63_2
;
; GFX9-LABEL: bitcast_v40i8_to_v20f16_scalar:
; GFX9: ; %bb.0:
@@ -25359,7 +26033,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s41, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s42, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
@@ -25425,8 +26099,17 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX9-NEXT: v_lshl_or_b32 v9, s5, 16, v8
; GFX9-NEXT: v_mov_b32_e32 v8, s4
-; GFX9-NEXT: s_cbranch_execnz .LBB63_3
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s63, s63, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s62
; GFX9-NEXT: v_mov_b32_e32 v6, 0xc0c0004
@@ -25528,11 +26211,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v6, v6, 16, v11
; GFX9-NEXT: v_lshl_or_b32 v7, v10, 16, v7
; GFX9-NEXT: v_lshl_or_b32 v9, v15, 16, v9
-; GFX9-NEXT: .LBB63_3: ; %end
+; GFX9-NEXT: .LBB63_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB63_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB63_2
;
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20f16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -25562,7 +26242,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-TRUE16-NEXT: s_and_b32 s59, s43, 0xff
@@ -25603,9 +26283,17 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s15, s5, v8
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, s59
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB63_3
-; GFX11-TRUE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB63_3
+; GFX11-TRUE16-NEXT: .LBB63_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s58, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB63_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
@@ -25677,11 +26365,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: .LBB63_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB63_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB63_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB63_2
;
; GFX11-FAKE16-LABEL: bitcast_v40i8_to_v20f16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -25711,7 +26396,7 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s58, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB63_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB63_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s59, s56, 0xff
@@ -25760,9 +26445,17 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s59
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v7, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v13, 16, v11
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB63_3
-; GFX11-FAKE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB63_3
+; GFX11-FAKE16-NEXT: .LBB63_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s58, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB63_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s57, s57, 3
; GFX11-FAKE16-NEXT: s_add_i32 s46, s46, 3
@@ -25844,11 +26537,8 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v11, 16, v7
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v8, v8, 16, v15
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v9, 16, v16
-; GFX11-FAKE16-NEXT: .LBB63_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB63_5: ; %end
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB63_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB63_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -26173,7 +26863,7 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB65_3
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -26205,8 +26895,17 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB65_4
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
@@ -26287,11 +26986,8 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: s_branch .LBB65_5
-; SI-NEXT: .LBB65_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB65_2
-; SI-NEXT: .LBB65_4:
+; SI-NEXT: s_branch .LBB65_6
+; SI-NEXT: .LBB65_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -26308,7 +27004,7 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB65_5: ; %end
+; SI-NEXT: .LBB65_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -26327,10 +27023,18 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB65_3
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_4
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s25, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -26383,9 +27087,7 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_3:
-; VI-NEXT: s_branch .LBB65_2
-; VI-NEXT: .LBB65_4:
+; VI-NEXT: .LBB65_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -26408,10 +27110,18 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_4
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
@@ -26424,9 +27134,7 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_3:
-; GFX9-NEXT: s_branch .LBB65_2
-; GFX9-NEXT: .LBB65_4:
+; GFX9-NEXT: .LBB65_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -26454,11 +27162,16 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_4
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
@@ -26470,8 +27183,6 @@ define inreg <5 x double> @bitcast_v20f16_to_v5f64_scalar(<20 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_3:
-; GFX11-NEXT: s_branch .LBB65_2
; GFX11-NEXT: .LBB65_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -26657,7 +27368,7 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB67_3
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s25, 16
; SI-NEXT: s_lshr_b32 s29, s23, 16
@@ -26669,8 +27380,26 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB67_4
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -26686,20 +27415,8 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; SI-NEXT: s_branch .LBB67_5
-; SI-NEXT: .LBB67_3:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB67_2
-; SI-NEXT: .LBB67_4:
+; SI-NEXT: s_branch .LBB67_6
+; SI-NEXT: .LBB67_5:
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: v_mov_b32_e32 v7, s23
; SI-NEXT: v_mov_b32_e32 v5, s21
@@ -26720,7 +27437,7 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v12, s8
; SI-NEXT: v_mov_b32_e32 v11, s6
; SI-NEXT: v_mov_b32_e32 v10, s4
-; SI-NEXT: .LBB67_5: ; %end
+; SI-NEXT: .LBB67_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
@@ -26757,19 +27474,25 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -26792,19 +27515,25 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -26832,19 +27561,22 @@ define inreg <20 x half> @bitcast_v5f64_to_v20f16_scalar(<5 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB67_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB67_3:
-; GFX11-NEXT: s_branch .LBB67_2
; GFX11-NEXT: .LBB67_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -27179,7 +27911,7 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB69_3
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -27211,8 +27943,17 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB69_4
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
@@ -27293,11 +28034,8 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: s_branch .LBB69_5
-; SI-NEXT: .LBB69_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB69_2
-; SI-NEXT: .LBB69_4:
+; SI-NEXT: s_branch .LBB69_6
+; SI-NEXT: .LBB69_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -27314,7 +28052,7 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB69_5: ; %end
+; SI-NEXT: .LBB69_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -27333,10 +28071,18 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB69_3
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB69_4
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s25, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -27389,9 +28135,7 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_3:
-; VI-NEXT: s_branch .LBB69_2
-; VI-NEXT: .LBB69_4:
+; VI-NEXT: .LBB69_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -27414,10 +28158,18 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB69_4
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s24, v0 op_sel_hi:[1,0]
@@ -27430,9 +28182,7 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_3:
-; GFX9-NEXT: s_branch .LBB69_2
-; GFX9-NEXT: .LBB69_4:
+; GFX9-NEXT: .LBB69_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -27460,11 +28210,16 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_4
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB69_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v7, 0x200, s19 op_sel_hi:[0,1]
@@ -27476,8 +28231,6 @@ define inreg <5 x i64> @bitcast_v20f16_to_v5i64_scalar(<20 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_3:
-; GFX11-NEXT: s_branch .LBB69_2
; GFX11-NEXT: .LBB69_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -27686,7 +28439,7 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s26, s25, 16
; SI-NEXT: s_lshr_b32 s27, s23, 16
@@ -27698,8 +28451,26 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[14:15], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[14:15], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr29
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s22, s22, 3
@@ -27720,7 +28491,7 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 16
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s12, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -27762,27 +28533,23 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s4
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr29
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v5i64_to_v20f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s24, s24, 3
; VI-NEXT: s_addc_u32 s25, s25, 0
; VI-NEXT: s_add_u32 s22, s22, 3
@@ -27793,7 +28560,7 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -27805,17 +28572,23 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v5i64_to_v20f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s24, s24, 3
; GFX9-NEXT: s_addc_u32 s25, s25, 0
; GFX9-NEXT: s_add_u32 s22, s22, 3
@@ -27826,7 +28599,7 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -27838,19 +28611,22 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-LABEL: bitcast_v5i64_to_v20f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB71_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s20, s20, 3
; GFX11-NEXT: s_addc_u32 s21, s21, 0
; GFX11-NEXT: s_add_u32 s18, s18, 3
@@ -27861,7 +28637,7 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB71_3: ; %end
+; GFX11-NEXT: .LBB71_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -27869,8 +28645,6 @@ define inreg <20 x half> @bitcast_v5i64_to_v20f16_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB71_4:
-; GFX11-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -29187,7 +29961,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s78, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s79, v0
-; SI-NEXT: s_cbranch_scc0 .LBB73_4
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -29279,8 +30053,17 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s45, s5
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB73_3
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -29411,7 +30194,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s45, s4, 0x3000000
-; SI-NEXT: .LBB73_3: ; %end
+; SI-NEXT: .LBB73_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -29435,9 +30218,6 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB73_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB73_2
;
; VI-LABEL: bitcast_v40i8_to_v5f64_scalar:
; VI: ; %bb.0:
@@ -29470,7 +30250,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s62, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s63, v0
-; VI-NEXT: s_cbranch_scc0 .LBB73_4
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -29533,8 +30313,17 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v9, s7, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB73_3
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -29636,11 +30425,8 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB73_3: ; %end
+; VI-NEXT: .LBB73_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB73_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB73_2
;
; GFX9-LABEL: bitcast_v40i8_to_v5f64_scalar:
; GFX9: ; %bb.0:
@@ -29673,7 +30459,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s62, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -29736,8 +30522,17 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v9, s7, v11, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: v_or_b32_e32 v9, v10, v9
-; GFX9-NEXT: s_cbranch_execnz .LBB73_3
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -29830,11 +30625,8 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v10
; GFX9-NEXT: v_add_u32_sdwa v9, v9, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB73_3: ; %end
+; GFX9-NEXT: .LBB73_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB73_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB73_2
;
; GFX11-LABEL: bitcast_v40i8_to_v5f64_scalar:
; GFX11: ; %bb.0:
@@ -29864,7 +30656,7 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s57, v0
; GFX11-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-NEXT: s_mov_b32 s58, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB73_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -29908,9 +30700,17 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v7, v12, v10
; GFX11-NEXT: v_or_b32_e32 v8, v14, v11
; GFX11-NEXT: v_or_b32_e32 v9, v15, v13
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_3
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB73_3
+; GFX11-NEXT: .LBB73_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-NEXT: .LBB73_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -30002,11 +30802,8 @@ define inreg <5 x double> @bitcast_v40i8_to_v5f64_scalar(<40 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_or_b32_e32 v8, v12, v13
; GFX11-NEXT: v_or_b32_e32 v9, v14, v15
-; GFX11-NEXT: .LBB73_3: ; %end
+; GFX11-NEXT: .LBB73_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB73_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-NEXT: s_branch .LBB73_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -30874,7 +31671,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_3
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s25, 24
; SI-NEXT: s_lshr_b32 s73, s25, 16
@@ -30906,8 +31703,46 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB75_4
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[15:16], s[18:19], 1.0
; SI-NEXT: v_lshr_b64 v[22:23], v[8:9], 8
@@ -30948,40 +31783,8 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; SI-NEXT: s_lshr_b32 s92, s17, 24
; SI-NEXT: s_lshr_b32 s93, s17, 16
; SI-NEXT: s_lshr_b32 s94, s17, 8
-; SI-NEXT: s_branch .LBB75_5
-; SI-NEXT: .LBB75_3:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB75_2
-; SI-NEXT: .LBB75_4:
+; SI-NEXT: s_branch .LBB75_6
+; SI-NEXT: .LBB75_5:
; SI-NEXT: v_mov_b32_e32 v20, s16
; SI-NEXT: v_mov_b32_e32 v15, s18
; SI-NEXT: v_mov_b32_e32 v8, s20
@@ -31002,7 +31805,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s6
; SI-NEXT: v_mov_b32_e32 v6, s4
; SI-NEXT: v_mov_b32_e32 v5, s10
-; SI-NEXT: .LBB75_5: ; %end
+; SI-NEXT: .LBB75_6: ; %end
; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v28
; SI-NEXT: s_and_b32 s4, s17, 0xff
@@ -31133,7 +31936,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB75_3
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s58, s25, 24
; VI-NEXT: s_lshr_b32 s57, s25, 16
@@ -31165,8 +31968,46 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB75_4
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[1:2], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[3:4], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[5:6], s[20:21], 1.0
@@ -31202,40 +32043,8 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v48, 8, v10
; VI-NEXT: v_lshrrev_b32_e32 v34, 16, v9
; VI-NEXT: v_lshrrev_b32_e32 v36, 8, v9
-; VI-NEXT: s_branch .LBB75_5
-; VI-NEXT: .LBB75_3:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: s_branch .LBB75_2
-; VI-NEXT: .LBB75_4:
+; VI-NEXT: s_branch .LBB75_6
+; VI-NEXT: .LBB75_5:
; VI-NEXT: v_mov_b32_e32 v9, s16
; VI-NEXT: v_mov_b32_e32 v7, s18
; VI-NEXT: v_mov_b32_e32 v5, s20
@@ -31276,7 +32085,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v13, s8
; VI-NEXT: v_mov_b32_e32 v12, s10
; VI-NEXT: v_mov_b32_e32 v11, s12
-; VI-NEXT: .LBB75_5: ; %end
+; VI-NEXT: .LBB75_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v15, v34, v15, s4
; VI-NEXT: v_perm_b32 v9, v9, v36, s4
@@ -31344,7 +32153,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s58, s25, 24
; GFX9-NEXT: s_lshr_b32 s57, s25, 16
@@ -31376,8 +32185,46 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB75_4
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[1:2], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[3:4], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[5:6], s[20:21], 1.0
@@ -31413,40 +32260,8 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 8, v10
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v9
-; GFX9-NEXT: s_branch .LBB75_5
-; GFX9-NEXT: .LBB75_3:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: s_branch .LBB75_2
-; GFX9-NEXT: .LBB75_4:
+; GFX9-NEXT: s_branch .LBB75_6
+; GFX9-NEXT: .LBB75_5:
; GFX9-NEXT: v_mov_b32_e32 v9, s16
; GFX9-NEXT: v_mov_b32_e32 v7, s18
; GFX9-NEXT: v_mov_b32_e32 v5, s20
@@ -31487,7 +32302,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s8
; GFX9-NEXT: v_mov_b32_e32 v12, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s12
-; GFX9-NEXT: .LBB75_5: ; %end
+; GFX9-NEXT: .LBB75_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v15, v33, v15, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v35, s4
@@ -31547,7 +32362,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB75_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s45, s21, 24
; GFX11-NEXT: s_lshr_b32 s44, s21, 16
@@ -31579,9 +32394,46 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB75_4
-; GFX11-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB75_3
+; GFX11-NEXT: .LBB75_2:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: .LBB75_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[5:6], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[1:2], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[7:8], s[2:3], 1.0
@@ -31617,40 +32469,8 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v48, 8, v13
; GFX11-NEXT: v_lshrrev_b32_e32 v34, 16, v12
; GFX11-NEXT: v_lshrrev_b32_e32 v35, 8, v12
-; GFX11-NEXT: s_branch .LBB75_5
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: s_branch .LBB75_2
-; GFX11-NEXT: .LBB75_4:
+; GFX11-NEXT: s_branch .LBB75_6
+; GFX11-NEXT: .LBB75_5:
; GFX11-NEXT: v_dual_mov_b32 v12, s0 :: v_dual_mov_b32 v7, s2
; GFX11-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v2, s21
; GFX11-NEXT: v_dual_mov_b32 v3, s18 :: v_dual_mov_b32 v4, s19
@@ -31671,7 +32491,7 @@ define inreg <40 x i8> @bitcast_v5f64_to_v40i8_scalar(<5 x double> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v36, s26 :: v_dual_mov_b32 v37, s25
; GFX11-NEXT: v_dual_mov_b32 v33, s24 :: v_dual_mov_b32 v38, s23
; GFX11-NEXT: v_dual_mov_b32 v39, s22 :: v_dual_mov_b32 v48, s15
-; GFX11-NEXT: .LBB75_5: ; %end
+; GFX11-NEXT: .LBB75_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v16, v34, v16, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v12, v12, v35, 0xc0c0004
@@ -33034,7 +33854,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s78, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s79, v0
-; SI-NEXT: s_cbranch_scc0 .LBB77_4
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -33126,8 +33946,17 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s45, s5
; SI-NEXT: s_or_b32 s45, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB77_3
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -33258,7 +34087,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s45, s4, 0x3000000
-; SI-NEXT: .LBB77_3: ; %end
+; SI-NEXT: .LBB77_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -33282,9 +34111,6 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB77_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB77_2
;
; VI-LABEL: bitcast_v40i8_to_v5i64_scalar:
; VI: ; %bb.0:
@@ -33317,7 +34143,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s62, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s63, v0
-; VI-NEXT: s_cbranch_scc0 .LBB77_4
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -33380,8 +34206,17 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v9, s7, v11, v9
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: s_cbranch_execnz .LBB77_3
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -33483,11 +34318,8 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x3000000, v7
; VI-NEXT: v_add_u32_e32 v8, vcc, 0x3000000, v8
; VI-NEXT: v_add_u32_e32 v9, vcc, 0x3000000, v9
-; VI-NEXT: .LBB77_3: ; %end
+; VI-NEXT: .LBB77_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB77_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB77_2
;
; GFX9-LABEL: bitcast_v40i8_to_v5i64_scalar:
; GFX9: ; %bb.0:
@@ -33520,7 +34352,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s62, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -33583,8 +34415,17 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v9, s7, v11, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX9-NEXT: v_or_b32_e32 v9, v10, v9
-; GFX9-NEXT: s_cbranch_execnz .LBB77_3
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -33677,11 +34518,8 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v10, 0x300, v10
; GFX9-NEXT: v_add_u32_sdwa v9, v9, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB77_3: ; %end
+; GFX9-NEXT: .LBB77_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB77_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB77_2
;
; GFX11-LABEL: bitcast_v40i8_to_v5i64_scalar:
; GFX11: ; %bb.0:
@@ -33711,7 +34549,7 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX11-NEXT: v_readfirstlane_b32 s57, v0
; GFX11-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-NEXT: s_mov_b32 s58, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB77_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -33755,9 +34593,17 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v7, v12, v10
; GFX11-NEXT: v_or_b32_e32 v8, v14, v11
; GFX11-NEXT: v_or_b32_e32 v9, v15, v13
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_3
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB77_3
+; GFX11-NEXT: .LBB77_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-NEXT: .LBB77_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v5, 0xc0c0004
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -33849,11 +34695,8 @@ define inreg <5 x i64> @bitcast_v40i8_to_v5i64_scalar(<40 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_or_b32_e32 v8, v12, v13
; GFX11-NEXT: v_or_b32_e32 v9, v14, v15
-; GFX11-NEXT: .LBB77_3: ; %end
+; GFX11-NEXT: .LBB77_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB77_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-NEXT: s_branch .LBB77_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -34752,7 +35595,7 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_4
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s25, 24
; SI-NEXT: s_lshr_b32 s73, s25, 16
@@ -34784,8 +35627,46 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB79_3
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s22, s22, 3
@@ -34826,7 +35707,7 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 8
-; SI-NEXT: .LBB79_3: ; %end
+; SI-NEXT: .LBB79_5: ; %end
; SI-NEXT: s_lshl_b32 s5, s60, 8
; SI-NEXT: s_and_b32 s7, s16, 0xff
; SI-NEXT: s_or_b32 s5, s7, s5
@@ -34956,44 +35837,12 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB79_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB79_2
;
; VI-LABEL: bitcast_v5i64_to_v40i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB79_4
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s26, s25, 24
; VI-NEXT: s_lshr_b32 s27, s25, 16
@@ -35025,8 +35874,46 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB79_3
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr29
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr28
+; VI-NEXT: ; implicit-def: $sgpr27
+; VI-NEXT: ; implicit-def: $sgpr26
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -35067,7 +35954,7 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s74, s17, 8
; VI-NEXT: s_lshr_b32 s75, s16, 16
; VI-NEXT: s_lshr_b32 s76, s16, 8
-; VI-NEXT: .LBB79_3: ; %end
+; VI-NEXT: .LBB79_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: v_mov_b32_e32 v1, s76
@@ -35150,44 +36037,12 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB79_4:
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr29
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr28
-; VI-NEXT: ; implicit-def: $sgpr27
-; VI-NEXT: ; implicit-def: $sgpr26
-; VI-NEXT: s_branch .LBB79_2
;
; GFX9-LABEL: bitcast_v5i64_to_v40i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s26, s25, 24
; GFX9-NEXT: s_lshr_b32 s27, s25, 16
@@ -35219,8 +36074,46 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB79_3
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr29
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr28
+; GFX9-NEXT: ; implicit-def: $sgpr27
+; GFX9-NEXT: ; implicit-def: $sgpr26
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -35261,7 +36154,7 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s74, s17, 8
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
; GFX9-NEXT: s_lshr_b32 s76, s16, 8
-; GFX9-NEXT: .LBB79_3: ; %end
+; GFX9-NEXT: .LBB79_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: v_mov_b32_e32 v1, s76
@@ -35335,45 +36228,13 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB79_4:
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr29
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr28
-; GFX9-NEXT: ; implicit-def: $sgpr27
-; GFX9-NEXT: ; implicit-def: $sgpr26
-; GFX9-NEXT: s_branch .LBB79_2
;
; GFX11-LABEL: bitcast_v5i64_to_v40i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s63, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB79_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s21, 24
; GFX11-NEXT: s_lshr_b32 s22, s21, 16
@@ -35405,9 +36266,46 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[10:11], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[12:13], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s63
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_3
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB79_3
+; GFX11-NEXT: .LBB79_2:
+; GFX11-NEXT: s_mov_b32 s63, -1
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr29
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr25
+; GFX11-NEXT: ; implicit-def: $sgpr27
+; GFX11-NEXT: ; implicit-def: $sgpr26
+; GFX11-NEXT: ; implicit-def: $sgpr23
+; GFX11-NEXT: ; implicit-def: $sgpr24
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr22
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB79_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s63, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -35448,7 +36346,7 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s60, s1, 8
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
; GFX11-NEXT: s_lshr_b32 s62, s0, 8
-; GFX11-NEXT: .LBB79_3: ; %end
+; GFX11-NEXT: .LBB79_5: ; %end
; GFX11-NEXT: v_mov_b32_e32 v6, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v7, s56, s10, v6
@@ -35496,38 +36394,6 @@ define inreg <40 x i8> @bitcast_v5i64_to_v40i8_scalar(<5 x i64> inreg %a, i32 in
; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
; GFX11-NEXT: scratch_store_b64 v0, v[9:10], off offset:32
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_4:
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr29
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr25
-; GFX11-NEXT: ; implicit-def: $sgpr27
-; GFX11-NEXT: ; implicit-def: $sgpr26
-; GFX11-NEXT: ; implicit-def: $sgpr23
-; GFX11-NEXT: ; implicit-def: $sgpr24
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr22
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB79_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -35640,19 +36506,25 @@ define inreg <5 x i64> @bitcast_v5f64_to_v5i64_scalar(<5 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -35675,19 +36547,25 @@ define inreg <5 x i64> @bitcast_v5f64_to_v5i64_scalar(<5 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -35710,19 +36588,25 @@ define inreg <5 x i64> @bitcast_v5f64_to_v5i64_scalar(<5 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -35750,19 +36634,22 @@ define inreg <5 x i64> @bitcast_v5f64_to_v5i64_scalar(<5 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -35908,10 +36795,18 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s26, 0
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -35922,7 +36817,7 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s23, s23, 0
; SI-NEXT: s_add_u32 s24, s24, 3
; SI-NEXT: s_addc_u32 s25, s25, 0
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -35934,17 +36829,23 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s24
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v5i64_to_v5f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s26, 0
-; VI-NEXT: s_cbranch_scc0 .LBB83_4
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_3
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -35955,7 +36856,7 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s23, s23, 0
; VI-NEXT: s_add_u32 s24, s24, 3
; VI-NEXT: s_addc_u32 s25, s25, 0
-; VI-NEXT: .LBB83_3: ; %end
+; VI-NEXT: .LBB83_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -35967,17 +36868,23 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v8, s24
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB83_4:
-; VI-NEXT: s_branch .LBB83_2
;
; GFX9-LABEL: bitcast_v5i64_to_v5f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s26, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_3
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -35988,7 +36895,7 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s23, s23, 0
; GFX9-NEXT: s_add_u32 s24, s24, 3
; GFX9-NEXT: s_addc_u32 s25, s25, 0
-; GFX9-NEXT: .LBB83_3: ; %end
+; GFX9-NEXT: .LBB83_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -36000,19 +36907,22 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v8, s24
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB83_4:
-; GFX9-NEXT: s_branch .LBB83_2
;
; GFX11-LABEL: bitcast_v5i64_to_v5f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s22, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB83_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB83_3
-; GFX11-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB83_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -36023,15 +36933,13 @@ define inreg <5 x double> @bitcast_v5i64_to_v5f64_scalar(<5 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s19, s19, 0
; GFX11-NEXT: s_add_u32 s20, s20, 3
; GFX11-NEXT: s_addc_u32 s21, s21, 0
-; GFX11-NEXT: .LBB83_3: ; %end
+; GFX11-NEXT: .LBB83_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB83_4:
-; GFX11-NEXT: s_branch .LBB83_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 123d1042e27c9..44c34161a1766 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -81,62 +81,82 @@ define inreg float @bitcast_i32_to_f32_scalar(i32 inreg %a, i32 inreg %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_i32_to_f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_i32_to_f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_i32_to_f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB1_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_3:
-; GFX11-NEXT: .LBB1_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -232,15 +252,21 @@ define inreg i32 @bitcast_f32_to_i32_scalar(float inreg %a, i32 inreg %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -248,15 +274,21 @@ define inreg i32 @bitcast_f32_to_i32_scalar(float inreg %a, i32 inreg %b) #0 {
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -264,15 +296,21 @@ define inreg i32 @bitcast_f32_to_i32_scalar(float inreg %a, i32 inreg %b) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -281,15 +319,18 @@ define inreg i32 @bitcast_f32_to_i32_scalar(float inreg %a, i32 inreg %b) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -393,68 +434,88 @@ define inreg <2 x i16> @bitcast_i32_to_v2i16_scalar(i32 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_i32_to_v2i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_i32_to_v2i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_i32_to_v2i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB5_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_3:
-; GFX11-NEXT: .LBB5_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -573,57 +634,75 @@ define inreg i32 @bitcast_v2i16_to_i32_scalar(<2 x i16> inreg %a, i32 inreg %b)
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s4, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v2i16_to_i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v2i16_to_i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -632,15 +711,18 @@ define inreg i32 @bitcast_v2i16_to_i32_scalar(<2 x i16> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -744,68 +826,88 @@ define inreg <2 x half> @bitcast_i32_to_v2f16_scalar(i32 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_i32_to_v2f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_i32_to_v2f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_i32_to_v2f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB9_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_3:
-; GFX11-NEXT: .LBB9_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -924,13 +1026,22 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -940,10 +1051,7 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -951,10 +1059,18 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -962,9 +1078,7 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -972,16 +1086,22 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -990,15 +1110,18 @@ define inreg i32 @bitcast_v2f16_to_i32_scalar(<2 x half> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1107,71 +1230,91 @@ define inreg <2 x bfloat> @bitcast_i32_to_v2bf16_scalar(i32 inreg %a, i32 inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_i32_to_v2bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_i32_to_v2bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_i32_to_v2bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB13_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_3:
-; GFX11-NEXT: .LBB13_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1385,32 +1528,46 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v2bf16_to_i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_3
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_4
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -1431,9 +1588,7 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_3:
-; VI-NEXT: s_branch .LBB15_2
-; VI-NEXT: .LBB15_4:
+; VI-NEXT: .LBB15_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -1441,10 +1596,18 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -1467,9 +1630,7 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -1478,11 +1639,16 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-TRUE16-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB15_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -1508,8 +1674,6 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB15_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB15_2
; GFX11-TRUE16-NEXT: .LBB15_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1519,11 +1683,16 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-FAKE16-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB15_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -1549,8 +1718,6 @@ define inreg i32 @bitcast_v2bf16_to_i32_scalar(<2 x bfloat> inreg %a, i32 inreg
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB15_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB15_2
; GFX11-FAKE16-NEXT: .LBB15_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1646,62 +1813,82 @@ define inreg <1 x i32> @bitcast_i32_to_v1i32_scalar(i32 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_i32_to_v1i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_i32_to_v1i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_i32_to_v1i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB17_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_3:
-; GFX11-NEXT: .LBB17_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1797,62 +1984,82 @@ define inreg i32 @bitcast_v1i32_to_i32_scalar(<1 x i32> inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_4
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB19_3
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB19_3: ; %end
+; SI-NEXT: .LBB19_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_4:
-; SI-NEXT: s_branch .LBB19_2
;
; VI-LABEL: bitcast_v1i32_to_i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_4
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_3
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB19_3: ; %end
+; VI-NEXT: .LBB19_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_4:
-; VI-NEXT: s_branch .LBB19_2
;
; GFX9-LABEL: bitcast_v1i32_to_i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_3
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB19_3: ; %end
+; GFX9-NEXT: .LBB19_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_4:
-; GFX9-NEXT: s_branch .LBB19_2
;
; GFX11-LABEL: bitcast_v1i32_to_i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB19_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: .LBB19_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB19_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -2044,111 +2251,134 @@ define inreg <4 x i8> @bitcast_i32_to_v4i8_scalar(i32 inreg %a, i32 inreg %b) #0
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 24
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_lshr_b32 s8, s16, 8
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_lshr_b32 s6, s16, 24
; SI-NEXT: s_lshr_b32 s8, s16, 8
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s7
; SI-NEXT: v_mov_b32_e32 v3, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_i32_to_v4i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s7, s16, 16
; VI-NEXT: s_lshr_b32 s8, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_lshr_b32 s7, s16, 16
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s8, s16, 8
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s8
; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_i32_to_v4i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s6, s16, 24
; GFX9-NEXT: s_lshr_b32 s7, s16, 16
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_lshr_b32 s7, s16, 16
; GFX9-NEXT: s_lshr_b32 s6, s16, 24
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_i32_to_v4i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-NEXT: s_lshr_b32 s2, s0, 16
; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB21_3
+; GFX11-NEXT: .LBB21_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: .LBB21_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s2, s0, 16
; GFX11-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2406,7 +2636,7 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -2417,8 +2647,17 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_or_b32 s6, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -2432,18 +2671,15 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s6, s4, 0x3000000
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v4i8_to_i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -2452,8 +2688,17 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -2465,17 +2710,14 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v4i8_to_i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -2484,8 +2726,17 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -2497,18 +2748,15 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v4i8_to_i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2517,9 +2765,17 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB23_3
+; GFX11-NEXT: .LBB23_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB23_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -2533,11 +2789,8 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2638,21 +2891,27 @@ define inreg <2 x i16> @bitcast_f32_to_v2i16_scalar(float inreg %a, i32 inreg %b
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -2662,15 +2921,21 @@ define inreg <2 x i16> @bitcast_f32_to_v2i16_scalar(float inreg %a, i32 inreg %b
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -2678,15 +2943,21 @@ define inreg <2 x i16> @bitcast_f32_to_v2i16_scalar(float inreg %a, i32 inreg %b
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -2695,15 +2966,18 @@ define inreg <2 x i16> @bitcast_f32_to_v2i16_scalar(float inreg %a, i32 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -2822,57 +3096,75 @@ define inreg float @bitcast_v2i16_to_f32_scalar(<2 x i16> inreg %a, i32 inreg %b
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s4, 0x30000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v2i16_to_f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v2i16_to_f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -2881,15 +3173,18 @@ define inreg float @bitcast_v2i16_to_f32_scalar(<2 x i16> inreg %a, i32 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -2993,21 +3288,27 @@ define inreg <2 x half> @bitcast_f32_to_v2f16_scalar(float inreg %a, i32 inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
@@ -3017,15 +3318,21 @@ define inreg <2 x half> @bitcast_f32_to_v2f16_scalar(float inreg %a, i32 inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3033,15 +3340,21 @@ define inreg <2 x half> @bitcast_f32_to_v2f16_scalar(float inreg %a, i32 inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -3050,15 +3363,18 @@ define inreg <2 x half> @bitcast_f32_to_v2f16_scalar(float inreg %a, i32 inreg %
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -3177,13 +3493,22 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_3
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_4
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -3193,10 +3518,7 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB31_2
-; SI-NEXT: .LBB31_4:
+; SI-NEXT: .LBB31_5:
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3204,10 +3526,18 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_3
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_4
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -3215,9 +3545,7 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_3:
-; VI-NEXT: s_branch .LBB31_2
-; VI-NEXT: .LBB31_4:
+; VI-NEXT: .LBB31_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3225,16 +3553,22 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -3243,15 +3577,18 @@ define inreg float @bitcast_v2f16_to_f32_scalar(<2 x half> inreg %a, i32 inreg %
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -3360,24 +3697,30 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -3388,15 +3731,21 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3404,15 +3753,21 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -3421,15 +3776,18 @@ define inreg <2 x bfloat> @bitcast_f32_to_v2bf16_scalar(float inreg %a, i32 inre
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -3643,32 +4001,46 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB35_4
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
-; SI-NEXT: s_cbranch_execnz .LBB35_3
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB35_3: ; %end
+; SI-NEXT: .LBB35_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_branch .LBB35_2
;
; VI-LABEL: bitcast_v2bf16_to_f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -3689,9 +4061,7 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3699,10 +4069,18 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -3725,9 +4103,7 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -3736,11 +4112,16 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-TRUE16-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB35_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -3766,8 +4147,6 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB35_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB35_2
; GFX11-TRUE16-NEXT: .LBB35_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -3777,11 +4156,16 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-FAKE16-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB35_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -3807,8 +4191,6 @@ define inreg float @bitcast_v2bf16_to_f32_scalar(<2 x bfloat> inreg %a, i32 inre
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB35_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB35_2
; GFX11-FAKE16-NEXT: .LBB35_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -3904,15 +4286,21 @@ define inreg <1 x i32> @bitcast_f32_to_v1i32_scalar(float inreg %a, i32 inreg %b
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3920,15 +4308,21 @@ define inreg <1 x i32> @bitcast_f32_to_v1i32_scalar(float inreg %a, i32 inreg %b
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3936,15 +4330,21 @@ define inreg <1 x i32> @bitcast_f32_to_v1i32_scalar(float inreg %a, i32 inreg %b
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -3953,15 +4353,18 @@ define inreg <1 x i32> @bitcast_f32_to_v1i32_scalar(float inreg %a, i32 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4057,62 +4460,82 @@ define inreg float @bitcast_v1i32_to_f32_scalar(<1 x i32> inreg %a, i32 inreg %b
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v1i32_to_f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v1i32_to_f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_3
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB39_3: ; %end
+; GFX9-NEXT: .LBB39_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_4:
-; GFX9-NEXT: s_branch .LBB39_2
;
; GFX11-LABEL: bitcast_v1i32_to_f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB39_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: .LBB39_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB39_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4304,24 +4727,30 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s7, s16, 24
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_lshr_b32 s8, s16, 8
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v3, s7
@@ -4332,24 +4761,30 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s7, s16, 24
; VI-NEXT: s_lshr_b32 s6, s16, 16
; VI-NEXT: s_lshr_b32 s8, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s8
; VI-NEXT: v_mov_b32_e32 v3, s7
@@ -4360,24 +4795,30 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s7, s16, 24
; GFX9-NEXT: s_lshr_b32 s6, s16, 16
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_mov_b32_e32 v3, s7
@@ -4388,29 +4829,34 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s0, 24
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB41_2
-; GFX11-NEXT: .LBB41_4:
+; GFX11-NEXT: .LBB41_5:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4669,7 +5115,7 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4680,8 +5126,17 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_or_b32 s6, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4695,18 +5150,15 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s6, s4, 0x3000000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v4i8_to_f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -4715,8 +5167,17 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -4728,17 +5189,14 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v4i8_to_f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -4747,8 +5205,17 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB43_3
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -4760,18 +5227,15 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB43_3: ; %end
+; GFX9-NEXT: .LBB43_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB43_2
;
; GFX11-LABEL: bitcast_v4i8_to_f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB43_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4780,9 +5244,17 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_3
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB43_3
+; GFX11-NEXT: .LBB43_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB43_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -4796,11 +5268,8 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB43_3: ; %end
+; GFX11-NEXT: .LBB43_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB43_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4913,61 +5382,79 @@ define inreg <2 x half> @bitcast_v2i16_to_v2f16_scalar(<2 x i16> inreg %a, i32 i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s5, s6, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_or_b32 s7, s4, s6
; SI-NEXT: s_and_b32 s6, s5, 0xffff
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v2i16_to_v2f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v2i16_to_v2f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -4976,15 +5463,18 @@ define inreg <2 x half> @bitcast_v2i16_to_v2f16_scalar(<2 x i16> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -5097,10 +5587,18 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -5109,13 +5607,11 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: v_mov_b32_e32 v1, s16
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
@@ -5125,10 +5621,18 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s4
@@ -5136,9 +5640,7 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -5146,16 +5648,22 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5164,15 +5672,18 @@ define inreg <2 x i16> @bitcast_v2f16_to_v2i16_scalar(<2 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -5280,59 +5791,77 @@ define inreg <2 x bfloat> @bitcast_v2i16_to_v2bf16_scalar(<2 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_4
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s6, s16, 16
; SI-NEXT: s_lshl_b32 s8, s7, 16
-; SI-NEXT: s_cbranch_execnz .LBB49_3
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_lshl_b32 s4, s7, 16
; SI-NEXT: s_lshl_b32 s5, s16, 16
; SI-NEXT: s_add_i32 s6, s5, 0x30000
; SI-NEXT: s_add_i32 s8, s4, 0x30000
-; SI-NEXT: .LBB49_3: ; %end
+; SI-NEXT: .LBB49_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB49_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB49_2
;
; VI-LABEL: bitcast_v2i16_to_v2bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_4
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_3
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB49_3: ; %end
+; VI-NEXT: .LBB49_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_4:
-; VI-NEXT: s_branch .LBB49_2
;
; GFX9-LABEL: bitcast_v2i16_to_v2bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5341,15 +5870,18 @@ define inreg <2 x bfloat> @bitcast_v2i16_to_v2bf16_scalar(<2 x i16> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB49_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: s_branch .LBB49_2
; GFX11-NEXT: .LBB49_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -5554,36 +6086,50 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v2bf16_to_v2i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_3
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_4
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -5604,9 +6150,7 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_3:
-; VI-NEXT: s_branch .LBB51_2
-; VI-NEXT: .LBB51_4:
+; VI-NEXT: .LBB51_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -5614,10 +6158,18 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -5639,9 +6191,7 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff0000
; GFX9-NEXT: v_and_or_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5650,11 +6200,16 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-TRUE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB51_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -5676,8 +6231,6 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB51_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB51_2
; GFX11-TRUE16-NEXT: .LBB51_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -5687,11 +6240,16 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-FAKE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB51_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -5715,8 +6273,6 @@ define inreg <2 x i16> @bitcast_v2bf16_to_v2i16_scalar(<2 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB51_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB51_2
; GFX11-FAKE16-NEXT: .LBB51_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -5835,57 +6391,75 @@ define inreg <1 x i32> @bitcast_v2i16_to_v1i32_scalar(<2 x i16> inreg %a, i32 in
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s7, s4, 0x30000
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v2i16_to_v1i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v2i16_to_v1i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -5894,15 +6468,18 @@ define inreg <1 x i32> @bitcast_v2i16_to_v1i32_scalar(<2 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_3:
-; GFX11-NEXT: s_branch .LBB53_2
; GFX11-NEXT: .LBB53_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -6006,68 +6583,88 @@ define inreg <2 x i16> @bitcast_v1i32_to_v2i16_scalar(<1 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_4
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB55_3
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: .LBB55_3: ; %end
+; SI-NEXT: .LBB55_5: ; %end
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB55_2
;
; VI-LABEL: bitcast_v1i32_to_v2i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_4
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_3
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB55_3: ; %end
+; VI-NEXT: .LBB55_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_4:
-; VI-NEXT: s_branch .LBB55_2
;
; GFX9-LABEL: bitcast_v1i32_to_v2i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_3
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB55_3: ; %end
+; GFX9-NEXT: .LBB55_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_4:
-; GFX9-NEXT: s_branch .LBB55_2
;
; GFX11-LABEL: bitcast_v1i32_to_v2i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB55_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: .LBB55_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB55_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -6275,15 +6872,26 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_lshr_b32 s8, s7, 8
; SI-NEXT: s_bfe_u32 s9, s6, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s5, s6, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
@@ -6292,30 +6900,37 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; SI-NEXT: s_lshr_b32 s8, s7, 8
; SI-NEXT: s_and_b32 s6, s5, 0xffff
; SI-NEXT: s_bfe_u32 s9, s5, 0x80008
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v2i16_to_v4i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s8, s16, 24
; VI-NEXT: s_lshr_b32 s6, s16, 16
; VI-NEXT: s_lshr_b32 s9, s16, 8
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_mov_b32 s7, s16
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s16, 16
; VI-NEXT: s_add_i32 s7, s16, 3
; VI-NEXT: s_add_i32 s6, s5, 3
@@ -6324,41 +6939,41 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_lshr_b32 s9, s4, 8
; VI-NEXT: s_bfe_u32 s8, s6, 0x80008
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s7
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v2i16_to_v4i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s7, s16, 24
; GFX9-NEXT: s_lshr_b32 s6, s16, 16
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_mov_b32_e32 v3, s7
@@ -6369,29 +6984,34 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB57_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s0, 24
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB57_3
+; GFX11-NEXT: .LBB57_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB57_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB57_2
-; GFX11-NEXT: .LBB57_4:
+; GFX11-NEXT: .LBB57_5:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6650,7 +7270,7 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -6662,8 +7282,18 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_or_b32 s6, s4, s6
; SI-NEXT: s_and_b32 s7, s5, 0xffff
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -6678,22 +7308,18 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_or_b32 s6, s4, s6
; SI-NEXT: s_and_b32 s7, s5, 0xffff
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v4i8_to_v2i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6702,8 +7328,17 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6715,17 +7350,14 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v4i8_to_v2i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6734,8 +7366,17 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB59_3
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6747,18 +7388,15 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB59_3: ; %end
+; GFX9-NEXT: .LBB59_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB59_2
;
; GFX11-LABEL: bitcast_v4i8_to_v2i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB59_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6767,9 +7405,17 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_3
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB59_3
+; GFX11-NEXT: .LBB59_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB59_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -6783,11 +7429,8 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB59_3: ; %end
+; GFX11-NEXT: .LBB59_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB59_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6906,12 +7549,22 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB61_3
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s7, s16, 16
; SI-NEXT: s_lshl_b32 s8, s6, 16
-; SI-NEXT: s_cbranch_execnz .LBB61_4
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -6920,15 +7573,11 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: s_branch .LBB61_5
-; SI-NEXT: .LBB61_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB61_2
-; SI-NEXT: .LBB61_4:
+; SI-NEXT: s_branch .LBB61_6
+; SI-NEXT: .LBB61_5:
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v0, s7
-; SI-NEXT: .LBB61_5: ; %end
+; SI-NEXT: .LBB61_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -6939,10 +7588,18 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB61_3
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_4
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s4
@@ -6950,9 +7607,7 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_3:
-; VI-NEXT: s_branch .LBB61_2
-; VI-NEXT: .LBB61_4:
+; VI-NEXT: .LBB61_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -6960,16 +7615,22 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_4
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_3:
-; GFX9-NEXT: s_branch .LBB61_2
-; GFX9-NEXT: .LBB61_4:
+; GFX9-NEXT: .LBB61_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -6978,15 +7639,18 @@ define inreg <2 x bfloat> @bitcast_v2f16_to_v2bf16_scalar(<2 x half> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_4
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_3:
-; GFX11-NEXT: s_branch .LBB61_2
; GFX11-NEXT: .LBB61_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7198,35 +7862,49 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB63_4
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: s_cbranch_execnz .LBB63_3
-; SI-NEXT: .LBB63_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
-; SI-NEXT: .LBB63_3: ; %end
+; SI-NEXT: .LBB63_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB63_4:
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: s_branch .LBB63_2
;
; VI-LABEL: bitcast_v2bf16_to_v2f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -7247,9 +7925,7 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -7257,10 +7933,18 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -7283,9 +7967,7 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -7294,11 +7976,16 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-TRUE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB63_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -7324,8 +8011,6 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB63_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB63_2
; GFX11-TRUE16-NEXT: .LBB63_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7335,11 +8020,16 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-FAKE16-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB63_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -7365,8 +8055,6 @@ define inreg <2 x half> @bitcast_v2bf16_to_v2f16_scalar(<2 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB63_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB63_2
; GFX11-FAKE16-NEXT: .LBB63_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -7485,13 +8173,22 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB65_3
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB65_4
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -7501,10 +8198,7 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB65_2
-; SI-NEXT: .LBB65_4:
+; SI-NEXT: .LBB65_5:
; SI-NEXT: v_mov_b32_e32 v0, s7
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -7512,10 +8206,18 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB65_3
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_4
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -7523,9 +8225,7 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_3:
-; VI-NEXT: s_branch .LBB65_2
-; VI-NEXT: .LBB65_4:
+; VI-NEXT: .LBB65_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -7533,16 +8233,22 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_4
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_3:
-; GFX9-NEXT: s_branch .LBB65_2
-; GFX9-NEXT: .LBB65_4:
+; GFX9-NEXT: .LBB65_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -7551,15 +8257,18 @@ define inreg <1 x i32> @bitcast_v2f16_to_v1i32_scalar(<2 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_4
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_3:
-; GFX11-NEXT: s_branch .LBB65_2
; GFX11-NEXT: .LBB65_4:
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7663,68 +8372,88 @@ define inreg <2 x half> @bitcast_v1i32_to_v2f16_scalar(<1 x i32> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB67_4
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB67_3
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s6, s16, 16
-; SI-NEXT: .LBB67_3: ; %end
+; SI-NEXT: .LBB67_5: ; %end
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB67_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB67_2
;
; VI-LABEL: bitcast_v1i32_to_v2f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB67_4
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_3
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB67_3: ; %end
+; VI-NEXT: .LBB67_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB67_4:
-; VI-NEXT: s_branch .LBB67_2
;
; GFX9-LABEL: bitcast_v1i32_to_v2f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_3
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB67_3: ; %end
+; GFX9-NEXT: .LBB67_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB67_4:
-; GFX9-NEXT: s_branch .LBB67_2
;
; GFX11-LABEL: bitcast_v1i32_to_v2f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB67_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB67_3:
-; GFX11-NEXT: .LBB67_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB67_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB67_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7931,15 +8660,26 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB69_3
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s7, s4, s5
; SI-NEXT: s_lshr_b32 s8, s7, 8
; SI-NEXT: s_bfe_u32 s9, s6, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB69_4
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
@@ -7951,12 +8691,7 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; SI-NEXT: v_bfe_u32 v3, v2, 8, 8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_3:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB69_2
-; SI-NEXT: .LBB69_4:
+; SI-NEXT: .LBB69_5:
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: v_mov_b32_e32 v0, s7
@@ -7967,13 +8702,24 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB69_3
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s8, s16, 16
; VI-NEXT: s_lshr_b32 s7, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB69_4
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_add_f16_e32 v2, s4, v0
@@ -7983,12 +8729,7 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; VI-NEXT: v_bfe_u32 v3, v2, 8, 8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_3:
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB69_2
-; VI-NEXT: .LBB69_4:
+; VI-NEXT: .LBB69_5:
; VI-NEXT: v_mov_b32_e32 v2, s8
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v3, s6
@@ -7999,25 +8740,31 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s7, s16, 24
; GFX9-NEXT: s_lshr_b32 s6, s16, 16
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB69_4
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_3:
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: s_branch .LBB69_2
-; GFX9-NEXT: .LBB69_4:
+; GFX9-NEXT: .LBB69_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_mov_b32_e32 v3, s7
@@ -8028,29 +8775,34 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_3
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s3, s0, 24
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_4
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_3:
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB69_2
-; GFX11-NEXT: .LBB69_4:
+; GFX11-NEXT: .LBB69_5:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8309,7 +9061,7 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -8321,8 +9073,18 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_or_b32 s6, s4, s6
; SI-NEXT: s_and_b32 s7, s5, 0xffff
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -8337,22 +9099,18 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_or_b32 s6, s4, s6
; SI-NEXT: s_and_b32 s7, s5, 0xffff
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v4i8_to_v2f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -8361,8 +9119,17 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -8374,17 +9141,14 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v4i8_to_v2f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -8393,8 +9157,17 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -8406,18 +9179,15 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-LABEL: bitcast_v4i8_to_v2f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8426,9 +9196,17 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -8442,11 +9220,8 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB71_3: ; %end
+; GFX11-NEXT: .LBB71_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB71_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8657,32 +9432,46 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB73_4
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v4
; SI-NEXT: v_lshr_b64 v[0:1], v[2:3], 16
-; SI-NEXT: s_cbranch_execnz .LBB73_3
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: .LBB73_3: ; %end
+; SI-NEXT: .LBB73_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB73_4:
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_branch .LBB73_2
;
; VI-LABEL: bitcast_v2bf16_to_v1i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -8703,9 +9492,7 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
+; VI-NEXT: .LBB73_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -8713,10 +9500,18 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -8739,9 +9534,7 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -8750,11 +9543,16 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-TRUE16-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: .LBB73_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -8780,8 +9578,6 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB73_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB73_2
; GFX11-TRUE16-NEXT: .LBB73_4:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -8791,11 +9587,16 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-FAKE16-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, -1
+; GFX11-FAKE16-NEXT: .LBB73_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -8821,8 +9622,6 @@ define inreg <1 x i32> @bitcast_v2bf16_to_v1i32_scalar(<2 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB73_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB73_2
; GFX11-FAKE16-NEXT: .LBB73_4:
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -8931,71 +9730,91 @@ define inreg <2 x bfloat> @bitcast_v1i32_to_v2bf16_scalar(<1 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_4
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB75_3
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: .LBB75_3: ; %end
+; SI-NEXT: .LBB75_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB75_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB75_2
;
; VI-LABEL: bitcast_v1i32_to_v2bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB75_4
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB75_3
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB75_3: ; %end
+; VI-NEXT: .LBB75_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB75_4:
-; VI-NEXT: s_branch .LBB75_2
;
; GFX9-LABEL: bitcast_v1i32_to_v2bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB75_3
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB75_3: ; %end
+; GFX9-NEXT: .LBB75_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB75_4:
-; GFX9-NEXT: s_branch .LBB75_2
;
; GFX11-LABEL: bitcast_v1i32_to_v2bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-NEXT: s_cbranch_vccz .LBB75_4
-; GFX11-NEXT: ; %bb.2: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: .LBB75_4: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s1, -1
+; GFX11-NEXT: .LBB75_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: .LBB75_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -9276,14 +10095,25 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; SI-NEXT: s_cmp_lg_u32 s17, 0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB77_4
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_lshr_b64 v[4:5], v[1:2], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: s_cbranch_execnz .LBB77_3
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr4
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
@@ -9292,27 +10122,33 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[4:5], v[1:2], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; SI-NEXT: .LBB77_3: ; %end
+; SI-NEXT: .LBB77_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v4
; SI-NEXT: v_mov_b32_e32 v1, v5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB77_4:
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: s_branch .LBB77_2
;
; VI-LABEL: bitcast_v2bf16_to_v4i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s8, s16, 16
; VI-NEXT: s_lshr_b32 s7, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x40c00000
; VI-NEXT: v_add_f32_e32 v1, s4, v0
@@ -9336,12 +10172,7 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v3, 24, v4
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v2, s8
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v3, s6
@@ -9352,13 +10183,24 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s6, s16, 24
; GFX9-NEXT: s_lshr_b32 s8, s16, 16
; GFX9-NEXT: s_lshr_b32 s7, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s16, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -9383,12 +10225,7 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s8
; GFX9-NEXT: v_mov_b32_e32 v3, s6
@@ -9399,15 +10236,25 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB77_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-TRUE16-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB77_3
+; GFX11-TRUE16-NEXT: .LBB77_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: .LBB77_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -9437,13 +10284,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB77_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: s_branch .LBB77_2
-; GFX11-TRUE16-NEXT: .LBB77_4:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: .LBB77_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -9451,15 +10293,25 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB77_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-FAKE16-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB77_3
+; GFX11-FAKE16-NEXT: .LBB77_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: .LBB77_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
@@ -9489,13 +10341,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB77_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-FAKE16-NEXT: s_branch .LBB77_2
-; GFX11-FAKE16-NEXT: .LBB77_4:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-FAKE16-NEXT: .LBB77_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s3
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
@@ -9752,7 +10599,7 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_4
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -9762,8 +10609,18 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s19, 24
; SI-NEXT: s_or_b32 s7, s5, s4
-; SI-NEXT: s_cbranch_execnz .LBB79_3
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_and_b32 s5, s18, 0xff
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -9776,22 +10633,18 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_add_i32 s6, s5, 0x3000000
; SI-NEXT: s_add_i32 s7, s4, 0x3000000
-; SI-NEXT: .LBB79_3: ; %end
+; SI-NEXT: .LBB79_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB79_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB79_2
;
; VI-LABEL: bitcast_v4i8_to_v2bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB79_4
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9800,8 +10653,17 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB79_3
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9813,17 +10675,14 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB79_3: ; %end
+; VI-NEXT: .LBB79_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB79_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB79_2
;
; GFX9-LABEL: bitcast_v4i8_to_v2bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9832,8 +10691,17 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB79_3
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9845,18 +10713,15 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB79_3: ; %end
+; GFX9-NEXT: .LBB79_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB79_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB79_2
;
; GFX11-LABEL: bitcast_v4i8_to_v2bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB79_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9865,9 +10730,17 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_3
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB79_3
+; GFX11-NEXT: .LBB79_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB79_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -9881,11 +10754,8 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB79_3: ; %end
+; GFX11-NEXT: .LBB79_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB79_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10074,111 +10944,134 @@ define inreg <4 x i8> @bitcast_v1i32_to_v4i8_scalar(<1 x i32> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s17, 0
-; SI-NEXT: s_cbranch_scc0 .LBB81_4
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s6, s16, 24
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_lshr_b32 s8, s16, 8
-; SI-NEXT: s_cbranch_execnz .LBB81_3
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_lshr_b32 s6, s16, 24
; SI-NEXT: s_lshr_b32 s8, s16, 8
-; SI-NEXT: .LBB81_3: ; %end
+; SI-NEXT: .LBB81_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s7
; SI-NEXT: v_mov_b32_e32 v3, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB81_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB81_2
;
; VI-LABEL: bitcast_v1i32_to_v4i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s17, 0
-; VI-NEXT: s_cbranch_scc0 .LBB81_4
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s7, s16, 16
; VI-NEXT: s_lshr_b32 s8, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB81_3
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr7
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_lshr_b32 s7, s16, 16
; VI-NEXT: s_lshr_b32 s6, s16, 24
; VI-NEXT: s_lshr_b32 s8, s16, 8
-; VI-NEXT: .LBB81_3: ; %end
+; VI-NEXT: .LBB81_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s8
; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB81_4:
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr7
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB81_2
;
; GFX9-LABEL: bitcast_v1i32_to_v4i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s17, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s6, s16, 24
; GFX9-NEXT: s_lshr_b32 s7, s16, 16
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB81_3
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr7
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_lshr_b32 s7, s16, 16
; GFX9-NEXT: s_lshr_b32 s6, s16, 24
; GFX9-NEXT: s_lshr_b32 s8, s16, 8
-; GFX9-NEXT: .LBB81_3: ; %end
+; GFX9-NEXT: .LBB81_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s8
; GFX9-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB81_4:
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr7
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB81_2
;
; GFX11-LABEL: bitcast_v1i32_to_v4i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB81_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-NEXT: s_lshr_b32 s2, s0, 16
; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_3
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB81_3
+; GFX11-NEXT: .LBB81_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: .LBB81_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s2, s0, 16
; GFX11-NEXT: s_lshr_b32 s1, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-NEXT: .LBB81_3: ; %end
+; GFX11-NEXT: .LBB81_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_4:
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: s_branch .LBB81_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10436,7 +11329,7 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s20, 0
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -10447,8 +11340,17 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_or_b32 s6, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -10462,18 +11364,15 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; SI-NEXT: s_or_b32 s5, s5, s6
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s6, s4, 0x3000000
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v4i8_to_v1i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s20, 0
-; VI-NEXT: s_cbranch_scc0 .LBB83_4
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -10482,8 +11381,17 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; VI-NEXT: v_perm_b32 v1, s18, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_cbranch_execnz .LBB83_3
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -10495,17 +11403,14 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
-; VI-NEXT: .LBB83_3: ; %end
+; VI-NEXT: .LBB83_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB83_4:
-; VI-NEXT: ; implicit-def: $vgpr0
-; VI-NEXT: s_branch .LBB83_2
;
; GFX9-LABEL: bitcast_v4i8_to_v1i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s20, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -10514,8 +11419,17 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_perm_b32 v1, s18, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB83_3
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -10527,18 +11441,15 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_add_u32_e32 v0, 0x300, v0
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB83_3: ; %end
+; GFX9-NEXT: .LBB83_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB83_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_branch .LBB83_2
;
; GFX11-LABEL: bitcast_v4i8_to_v1i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB83_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB83_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10547,9 +11458,17 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB83_3
-; GFX11-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB83_3
+; GFX11-NEXT: .LBB83_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB83_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -10563,11 +11482,8 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB83_3: ; %end
+; GFX11-NEXT: .LBB83_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB83_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: s_branch .LBB83_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
index 79c9fc7faf339..8b5acdca0294f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.352bit.ll
@@ -125,10 +125,18 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s24, s24, 3
@@ -140,7 +148,7 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -153,17 +161,23 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v11i32_to_v11f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
@@ -175,7 +189,7 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -188,17 +202,23 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v11i32_to_v11f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
@@ -210,7 +230,7 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -223,19 +243,22 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v11i32_to_v11f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
@@ -247,7 +270,7 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -256,8 +279,6 @@ define inreg <11 x float> @bitcast_v11i32_to_v11f32_scalar(<11 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_mov_b32_e32 v10, s22
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -388,10 +409,18 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
; SI-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -404,9 +433,7 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -424,10 +451,18 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -440,9 +475,7 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -460,10 +493,18 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -476,9 +517,7 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -501,11 +540,16 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
@@ -518,8 +562,6 @@ define inreg <11 x i32> @bitcast_v11f32_to_v11i32_scalar(<11 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -733,7 +775,7 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s27, s25, 16
; SI-NEXT: s_lshr_b32 s40, s23, 16
@@ -746,8 +788,27 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s17, s17, 3
@@ -770,7 +831,7 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s42, s19, 16
; SI-NEXT: s_lshr_b32 s43, s17, 16
; SI-NEXT: s_lshr_b64 s[8:9], s[26:27], 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -816,28 +877,23 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v9, s14
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v11i32_to_v22i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
@@ -849,7 +905,7 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -862,17 +918,23 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v11i32_to_v22i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
@@ -884,7 +946,7 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -897,19 +959,22 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v11i32_to_v22i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
@@ -921,7 +986,7 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -930,8 +995,6 @@ define inreg <22 x i16> @bitcast_v11i32_to_v22i16_scalar(<11 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_mov_b32_e32 v10, s22
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1221,7 +1284,7 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s56, s17, 16
; SI-NEXT: s_lshr_b32 s57, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -1256,8 +1319,17 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s14, s26, 0xffff
; SI-NEXT: s_lshl_b32 s27, s15, 16
; SI-NEXT: s_or_b32 s14, s14, s27
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s27, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s27, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -1313,7 +1385,7 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -1326,18 +1398,23 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v22i16_to_v11i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s26, 3
; VI-NEXT: s_and_b32 s4, s26, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1393,7 +1470,7 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1406,17 +1483,23 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v22i16_to_v11i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
@@ -1429,9 +1512,7 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1454,11 +1535,16 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
@@ -1471,8 +1557,6 @@ define inreg <11 x i32> @bitcast_v22i16_to_v11i32_scalar(<22 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -1686,7 +1770,7 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s27, s25, 16
; SI-NEXT: s_lshr_b32 s40, s23, 16
@@ -1699,8 +1783,27 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s17, s17, 3
@@ -1723,7 +1826,7 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s42, s19, 16
; SI-NEXT: s_lshr_b32 s43, s17, 16
; SI-NEXT: s_lshr_b64 s[8:9], s[26:27], 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1769,28 +1872,23 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v9, s14
; SI-NEXT: v_mov_b32_e32 v10, s8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v11i32_to_v22f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s24, s24, 3
@@ -1802,7 +1900,7 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1815,17 +1913,23 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v11i32_to_v22f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
; GFX9-NEXT: s_add_i32 s24, s24, 3
@@ -1837,7 +1941,7 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1850,19 +1954,22 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v9, s25
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v11i32_to_v22f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
; GFX11-NEXT: s_add_i32 s20, s20, 3
@@ -1874,7 +1981,7 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1883,8 +1990,6 @@ define inreg <22 x half> @bitcast_v11i32_to_v22f16_scalar(<11 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_mov_b32_e32 v10, s22
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2218,7 +2323,7 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s56, s17, 16
; SI-NEXT: s_lshr_b32 s57, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -2253,8 +2358,17 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; SI-NEXT: s_and_b32 s14, s26, 0xffff
; SI-NEXT: s_lshl_b32 s27, s15, 16
; SI-NEXT: s_or_b32 s14, s14, s27
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s27, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s27, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s57
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s56
@@ -2344,10 +2458,7 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2365,10 +2476,18 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s26, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2426,9 +2545,7 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v11
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2446,10 +2563,18 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
@@ -2463,9 +2588,7 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2488,11 +2611,16 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
@@ -2505,8 +2633,6 @@ define inreg <11 x i32> @bitcast_v22f16_to_v11i32_scalar(<22 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2714,7 +2840,7 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_3
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s27, s17, 16
; SI-NEXT: s_lshr_b32 s43, s25, 16
@@ -2727,8 +2853,27 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_4
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -2751,21 +2896,8 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
-; SI-NEXT: s_branch .LBB13_5
-; SI-NEXT: .LBB13_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB13_2
-; SI-NEXT: .LBB13_4:
+; SI-NEXT: s_branch .LBB13_6
+; SI-NEXT: .LBB13_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2788,7 +2920,7 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v15, s10
; SI-NEXT: v_mov_b32_e32 v12, s6
; SI-NEXT: v_mov_b32_e32 v11, s4
-; SI-NEXT: .LBB13_5: ; %end
+; SI-NEXT: .LBB13_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v14
@@ -2828,10 +2960,18 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_3
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_4
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -2844,9 +2984,7 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_3:
-; VI-NEXT: s_branch .LBB13_2
-; VI-NEXT: .LBB13_4:
+; VI-NEXT: .LBB13_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2869,10 +3007,18 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_4
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -2885,9 +3031,7 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_3:
-; GFX9-NEXT: s_branch .LBB13_2
-; GFX9-NEXT: .LBB13_4:
+; GFX9-NEXT: .LBB13_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2915,11 +3059,16 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_4
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
@@ -2932,8 +3081,6 @@ define inreg <22 x i16> @bitcast_v11f32_to_v22i16_scalar(<11 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_3:
-; GFX11-NEXT: s_branch .LBB13_2
; GFX11-NEXT: .LBB13_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3233,7 +3380,7 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s56, s17, 16
; SI-NEXT: s_lshr_b32 s57, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -3268,8 +3415,17 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; SI-NEXT: s_and_b32 s14, s26, 0xffff
; SI-NEXT: s_lshl_b32 s27, s15, 16
; SI-NEXT: s_or_b32 s14, s14, s27
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s27, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s27, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -3325,7 +3481,7 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -3338,18 +3494,23 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v9, s13
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v22i16_to_v11f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s26, 3
; VI-NEXT: s_and_b32 s4, s26, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -3405,7 +3566,7 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3418,17 +3579,23 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v22i16_to_v11f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
@@ -3441,9 +3608,7 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3466,11 +3631,16 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
@@ -3483,8 +3653,6 @@ define inreg <11 x float> @bitcast_v22i16_to_v11f32_scalar(<22 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3692,7 +3860,7 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_3
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s27, s17, 16
; SI-NEXT: s_lshr_b32 s43, s25, 16
@@ -3705,8 +3873,27 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_4
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v5, s21, 1.0
; SI-NEXT: v_add_f32_e64 v4, s20, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -3729,21 +3916,8 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v1
-; SI-NEXT: s_branch .LBB17_5
-; SI-NEXT: .LBB17_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB17_2
-; SI-NEXT: .LBB17_4:
+; SI-NEXT: s_branch .LBB17_6
+; SI-NEXT: .LBB17_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3766,7 +3940,7 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v15, s10
; SI-NEXT: v_mov_b32_e32 v12, s6
; SI-NEXT: v_mov_b32_e32 v11, s4
-; SI-NEXT: .LBB17_5: ; %end
+; SI-NEXT: .LBB17_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v14
@@ -3806,10 +3980,18 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_3
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_4
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
; VI-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -3822,9 +4004,7 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_3:
-; VI-NEXT: s_branch .LBB17_2
-; VI-NEXT: .LBB17_4:
+; VI-NEXT: .LBB17_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3847,10 +4027,18 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_4
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s24, 1.0
@@ -3863,9 +4051,7 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_3:
-; GFX9-NEXT: s_branch .LBB17_2
-; GFX9-NEXT: .LBB17_4:
+; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3893,11 +4079,16 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_4
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s20, 1.0
@@ -3910,8 +4101,6 @@ define inreg <22 x half> @bitcast_v11f32_to_v22f16_scalar(<11 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_3:
-; GFX11-NEXT: s_branch .LBB17_2
; GFX11-NEXT: .LBB17_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4255,7 +4444,7 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s56, s17, 16
; SI-NEXT: s_lshr_b32 s57, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s57, 16
@@ -4290,8 +4479,17 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; SI-NEXT: s_and_b32 s14, s26, 0xffff
; SI-NEXT: s_lshl_b32 s27, s15, 16
; SI-NEXT: s_or_b32 s14, s14, s27
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s27, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s27, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s57
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s56
@@ -4381,10 +4579,7 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -4402,10 +4597,18 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s26, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -4463,9 +4666,7 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v11
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4483,10 +4684,18 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
@@ -4500,9 +4709,7 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4525,11 +4732,16 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
@@ -4542,8 +4754,6 @@ define inreg <11 x float> @bitcast_v22f16_to_v11f32_scalar(<22 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4890,7 +5100,7 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s57, s17, 16
; SI-NEXT: s_lshr_b32 s62, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s57, 16
@@ -4935,8 +5145,27 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 16
; SI-NEXT: s_mov_b32 s5, s45
; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr27
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s74, 16
@@ -5003,7 +5232,7 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s60, s7, 16
; SI-NEXT: s_lshr_b32 s61, s5, 16
; SI-NEXT: s_lshr_b32 s56, s27, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -5049,28 +5278,23 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v9, s5
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v22i16_to_v22f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
@@ -5126,7 +5350,7 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5139,17 +5363,23 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v9, s25
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v22i16_to_v22f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s24, 3 op_sel_hi:[1,0]
@@ -5162,9 +5392,7 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5192,11 +5420,16 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s20, 3 op_sel_hi:[1,0]
@@ -5209,8 +5442,6 @@ define inreg <22 x half> @bitcast_v22i16_to_v22f16_scalar(<22 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -5518,10 +5749,18 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s28, s16, 16
; SI-NEXT: s_cmp_lg_u32 s27, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s14
; SI-NEXT: v_cvt_f32_f16_e32 v5, s11
@@ -5615,10 +5854,8 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[14:15], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[10:11], v[8:9], 16
-; SI-NEXT: s_branch .LBB23_5
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: s_branch .LBB23_6
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v20, s12
; SI-NEXT: v_mov_b32_e32 v24, s9
; SI-NEXT: v_mov_b32_e32 v21, s10
@@ -5641,7 +5878,7 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v14, s14
; SI-NEXT: v_mov_b32_e32 v12, s13
; SI-NEXT: v_mov_b32_e32 v10, s11
-; SI-NEXT: .LBB23_5: ; %end
+; SI-NEXT: .LBB23_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v18
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -5681,10 +5918,18 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s27, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s25, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s5
@@ -5742,9 +5987,7 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v0, v1
; VI-NEXT: v_or_b32_e32 v0, v11, v12
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5767,10 +6010,18 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s27, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v9, s25, v0 op_sel_hi:[1,0]
@@ -5784,9 +6035,7 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5814,11 +6063,16 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s23, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s20 op_sel_hi:[0,1]
@@ -5831,8 +6085,6 @@ define inreg <22 x i16> @bitcast_v22f16_to_v22i16_scalar(<22 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
index 6d91f0ea85573..369bec7445f6c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.384bit.ll
@@ -129,10 +129,18 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s25, s25, 3
@@ -145,7 +153,7 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -159,17 +167,23 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v12i32_to_v12f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
@@ -182,7 +196,7 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -196,17 +210,23 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v12i32_to_v12f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s27, s27, 3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
@@ -219,7 +239,7 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -233,19 +253,22 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v12i32_to_v12f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s23, s23, 3
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
@@ -258,7 +281,7 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -267,8 +290,6 @@ define inreg <12 x float> @bitcast_v12i32_to_v12f32_scalar(<12 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -402,10 +423,18 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -419,9 +448,7 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -440,10 +467,18 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -457,9 +492,7 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -478,10 +511,18 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -495,9 +536,7 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -521,11 +560,16 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
@@ -539,8 +583,6 @@ define inreg <12 x i32> @bitcast_v12f32_to_v12i32_scalar(<12 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -689,10 +731,18 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s25, s25, 3
@@ -705,7 +755,7 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -719,17 +769,23 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v12i32_to_v6f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
@@ -742,7 +798,7 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -756,17 +812,23 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v12i32_to_v6f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s27, s27, 3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
@@ -779,7 +841,7 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -793,19 +855,22 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v12i32_to_v6f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s23, s23, 3
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
@@ -818,7 +883,7 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -827,8 +892,6 @@ define inreg <6 x double> @bitcast_v12i32_to_v6f64_scalar(<12 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -945,10 +1008,18 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_3
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_4
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -956,9 +1027,7 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_3:
-; SI-NEXT: s_branch .LBB7_2
-; SI-NEXT: .LBB7_4:
+; SI-NEXT: .LBB7_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -977,10 +1046,18 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_3
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_4
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -988,9 +1065,7 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_3:
-; VI-NEXT: s_branch .LBB7_2
-; VI-NEXT: .LBB7_4:
+; VI-NEXT: .LBB7_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1009,10 +1084,18 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -1020,9 +1103,7 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1046,11 +1127,16 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
@@ -1058,8 +1144,6 @@ define inreg <12 x i32> @bitcast_v6f64_to_v12i32_scalar(<6 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -1208,10 +1292,18 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s25, s25, 3
@@ -1224,7 +1316,7 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1238,17 +1330,23 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v12i32_to_v6i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
@@ -1261,7 +1359,7 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1275,17 +1373,23 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v12i32_to_v6i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s27, s27, 3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
@@ -1298,7 +1402,7 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1312,19 +1416,22 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v12i32_to_v6i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s23, s23, 3
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
@@ -1337,7 +1444,7 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1346,8 +1453,6 @@ define inreg <6 x i64> @bitcast_v12i32_to_v6i64_scalar(<12 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1491,10 +1596,18 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_4
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_3
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s26, s26, 3
; SI-NEXT: s_addc_u32 s27, s27, 0
; SI-NEXT: s_add_u32 s24, s24, 3
@@ -1507,7 +1620,7 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB11_3: ; %end
+; SI-NEXT: .LBB11_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1521,17 +1634,23 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_4:
-; SI-NEXT: s_branch .LBB11_2
;
; VI-LABEL: bitcast_v6i64_to_v12i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_4
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_3
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s26, s26, 3
; VI-NEXT: s_addc_u32 s27, s27, 0
; VI-NEXT: s_add_u32 s24, s24, 3
@@ -1544,7 +1663,7 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB11_3: ; %end
+; VI-NEXT: .LBB11_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1558,17 +1677,23 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_4:
-; VI-NEXT: s_branch .LBB11_2
;
; GFX9-LABEL: bitcast_v6i64_to_v12i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_3
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s26, s26, 3
; GFX9-NEXT: s_addc_u32 s27, s27, 0
; GFX9-NEXT: s_add_u32 s24, s24, 3
@@ -1581,7 +1706,7 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB11_3: ; %end
+; GFX9-NEXT: .LBB11_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1595,19 +1720,22 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_4:
-; GFX9-NEXT: s_branch .LBB11_2
;
; GFX11-LABEL: bitcast_v6i64_to_v12i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_3
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s22, s22, 3
; GFX11-NEXT: s_addc_u32 s23, s23, 0
; GFX11-NEXT: s_add_u32 s20, s20, 3
@@ -1620,7 +1748,7 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB11_3: ; %end
+; GFX11-NEXT: .LBB11_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1629,8 +1757,6 @@ define inreg <12 x i32> @bitcast_v6i64_to_v12i32_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_4:
-; GFX11-NEXT: s_branch .LBB11_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1846,7 +1972,7 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s27, 16
; SI-NEXT: s_lshr_b32 s41, s25, 16
@@ -1860,8 +1986,28 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -1886,7 +2032,7 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s43, s21, 16
; SI-NEXT: s_lshr_b32 s44, s19, 16
; SI-NEXT: s_lshr_b32 s45, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -1936,29 +2082,23 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v10, s4
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v12i32_to_v24i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
@@ -1971,7 +2111,7 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1985,17 +2125,23 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v12i32_to_v24i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s27, s27, 3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
@@ -2008,7 +2154,7 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2022,19 +2168,22 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v12i32_to_v24i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s23, s23, 3
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
@@ -2047,7 +2196,7 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2056,8 +2205,6 @@ define inreg <24 x i16> @bitcast_v12i32_to_v24i16_scalar(<12 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2364,7 +2511,7 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s58, s17, 16
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -2402,8 +2549,17 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s15, s27, 0xffff
; SI-NEXT: s_lshl_b32 s28, s40, 16
; SI-NEXT: s_or_b32 s15, s15, s28
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s28, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s28, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -2464,7 +2620,7 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2478,18 +2634,23 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v24i16_to_v12i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s27, 3
; VI-NEXT: s_and_b32 s4, s27, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2550,7 +2711,7 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2564,17 +2725,23 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v24i16_to_v12i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
@@ -2588,9 +2755,7 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2614,11 +2779,16 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
@@ -2632,8 +2802,6 @@ define inreg <12 x i32> @bitcast_v24i16_to_v12i32_scalar(<24 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2857,7 +3025,7 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s27, 16
; SI-NEXT: s_lshr_b32 s41, s25, 16
@@ -2871,8 +3039,28 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -2897,7 +3085,7 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s43, s21, 16
; SI-NEXT: s_lshr_b32 s44, s19, 16
; SI-NEXT: s_lshr_b32 s45, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -2947,29 +3135,23 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s4
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v12i32_to_v24f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s25, s25, 3
@@ -2982,7 +3164,7 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2996,17 +3178,23 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v12i32_to_v24f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s27, s27, 3
; GFX9-NEXT: s_add_i32 s26, s26, 3
; GFX9-NEXT: s_add_i32 s25, s25, 3
@@ -3019,7 +3207,7 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3033,19 +3221,22 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v12i32_to_v24f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s23, s23, 3
; GFX11-NEXT: s_add_i32 s22, s22, 3
; GFX11-NEXT: s_add_i32 s21, s21, 3
@@ -3058,7 +3249,7 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3067,8 +3258,6 @@ define inreg <24 x half> @bitcast_v12i32_to_v24f16_scalar(<12 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3423,7 +3612,7 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s58, s17, 16
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -3461,8 +3650,17 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; SI-NEXT: s_and_b32 s15, s27, 0xffff
; SI-NEXT: s_lshl_b32 s28, s40, 16
; SI-NEXT: s_or_b32 s15, s15, s28
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s28, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s28, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -3560,10 +3758,7 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -3582,10 +3777,18 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s27, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -3648,9 +3851,7 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v12
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3669,10 +3870,18 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v11, s27, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
@@ -3687,9 +3896,7 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3713,11 +3920,16 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
@@ -3731,8 +3943,6 @@ define inreg <12 x i32> @bitcast_v24f16_to_v12i32_scalar(<24 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3874,10 +4084,18 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -3891,9 +4109,7 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -3916,10 +4132,18 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -3933,9 +4157,7 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3958,10 +4180,18 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -3975,9 +4205,7 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4005,11 +4233,16 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
@@ -4023,8 +4256,6 @@ define inreg <6 x double> @bitcast_v12f32_to_v6f64_scalar(<12 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4151,10 +4382,18 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_3
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_4
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -4162,9 +4401,7 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_3:
-; SI-NEXT: s_branch .LBB23_2
-; SI-NEXT: .LBB23_4:
+; SI-NEXT: .LBB23_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4183,10 +4420,18 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -4194,9 +4439,7 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4215,10 +4458,18 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -4226,9 +4477,7 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4252,11 +4501,16 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
@@ -4264,8 +4518,6 @@ define inreg <12 x float> @bitcast_v6f64_to_v12f32_scalar(<6 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_3:
-; GFX11-NEXT: s_branch .LBB23_2
; GFX11-NEXT: .LBB23_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4407,10 +4659,18 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -4424,9 +4684,7 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4449,10 +4707,18 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -4466,9 +4732,7 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4491,10 +4755,18 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -4508,9 +4780,7 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4538,11 +4808,16 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
@@ -4556,8 +4831,6 @@ define inreg <6 x i64> @bitcast_v12f32_to_v6i64_scalar(<12 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4711,10 +4984,18 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s26, s26, 3
; SI-NEXT: s_addc_u32 s27, s27, 0
; SI-NEXT: s_add_u32 s24, s24, 3
@@ -4727,7 +5008,7 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4741,17 +5022,23 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v6i64_to_v12f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s26, s26, 3
; VI-NEXT: s_addc_u32 s27, s27, 0
; VI-NEXT: s_add_u32 s24, s24, 3
@@ -4764,7 +5051,7 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4778,17 +5065,23 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v6i64_to_v12f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_3
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s26, s26, 3
; GFX9-NEXT: s_addc_u32 s27, s27, 0
; GFX9-NEXT: s_add_u32 s24, s24, 3
@@ -4801,7 +5094,7 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB27_3: ; %end
+; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4815,19 +5108,22 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_4:
-; GFX9-NEXT: s_branch .LBB27_2
;
; GFX11-LABEL: bitcast_v6i64_to_v12f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s22, s22, 3
; GFX11-NEXT: s_addc_u32 s23, s23, 0
; GFX11-NEXT: s_add_u32 s20, s20, 3
@@ -4840,7 +5136,7 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB27_3: ; %end
+; GFX11-NEXT: .LBB27_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4849,8 +5145,6 @@ define inreg <12 x float> @bitcast_v6i64_to_v12f32_scalar(<6 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_4:
-; GFX11-NEXT: s_branch .LBB27_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5059,7 +5353,7 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s45, s27, 16
; SI-NEXT: s_lshr_b32 s44, s25, 16
@@ -5073,8 +5367,28 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -5099,22 +5413,8 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5139,7 +5439,7 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v14, s8
; SI-NEXT: v_mov_b32_e32 v13, s6
; SI-NEXT: v_mov_b32_e32 v12, s4
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -5182,10 +5482,18 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -5199,9 +5507,7 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5224,10 +5530,18 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -5241,9 +5555,7 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5271,11 +5583,16 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
@@ -5289,8 +5606,6 @@ define inreg <24 x i16> @bitcast_v12f32_to_v24i16_scalar(<12 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -5607,7 +5922,7 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s58, s17, 16
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -5645,8 +5960,17 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; SI-NEXT: s_and_b32 s15, s27, 0xffff
; SI-NEXT: s_lshl_b32 s28, s40, 16
; SI-NEXT: s_or_b32 s15, s15, s28
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s28, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s28, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -5707,7 +6031,7 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -5721,18 +6045,23 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v24i16_to_v12f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s27, 3
; VI-NEXT: s_and_b32 s4, s27, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -5793,7 +6122,7 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5807,17 +6136,23 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v24i16_to_v12f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
@@ -5831,9 +6166,7 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5857,11 +6190,16 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
@@ -5875,8 +6213,6 @@ define inreg <12 x float> @bitcast_v24i16_to_v12f32_scalar(<24 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -6093,7 +6429,7 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s45, s27, 16
; SI-NEXT: s_lshr_b32 s44, s25, 16
@@ -6107,8 +6443,28 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
; SI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -6133,22 +6489,8 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6173,7 +6515,7 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v14, s8
; SI-NEXT: v_mov_b32_e32 v13, s6
; SI-NEXT: v_mov_b32_e32 v12, s4
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -6216,10 +6558,18 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
; VI-NEXT: v_add_f32_e64 v10, s26, 1.0
; VI-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -6233,9 +6583,7 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6258,10 +6606,18 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v10, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v9, s25, 1.0
@@ -6275,9 +6631,7 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6305,11 +6659,16 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
; GFX11-NEXT: v_add_f32_e64 v10, s22, 1.0
; GFX11-NEXT: v_add_f32_e64 v9, s21, 1.0
@@ -6323,8 +6682,6 @@ define inreg <24 x half> @bitcast_v12f32_to_v24f16_scalar(<12 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -6689,7 +7046,7 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s58, s17, 16
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -6727,8 +7084,17 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; SI-NEXT: s_and_b32 s15, s27, 0xffff
; SI-NEXT: s_lshl_b32 s28, s40, 16
; SI-NEXT: s_or_b32 s15, s15, s28
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s28, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s28, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -6826,10 +7192,7 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
; SI-NEXT: v_or_b32_e32 v11, v13, v11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -6848,10 +7211,18 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s27, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -6914,9 +7285,7 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v12
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6935,10 +7304,18 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v11, s27, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
@@ -6953,9 +7330,7 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6979,11 +7354,16 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
@@ -6997,8 +7377,6 @@ define inreg <12 x float> @bitcast_v24f16_to_v12f32_scalar(<24 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -7123,10 +7501,18 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -7134,9 +7520,7 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -7159,10 +7543,18 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -7170,9 +7562,7 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7195,10 +7585,18 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -7206,9 +7604,7 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7236,11 +7632,16 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
@@ -7248,8 +7649,6 @@ define inreg <6 x i64> @bitcast_v6f64_to_v6i64_scalar(<6 x double> inreg %a, i32
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -7403,10 +7802,18 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -7419,7 +7826,7 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s25, s25, 0
; SI-NEXT: s_add_u32 s26, s26, 3
; SI-NEXT: s_addc_u32 s27, s27, 0
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -7433,17 +7840,23 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s26
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v6i64_to_v6f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -7456,7 +7869,7 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s25, s25, 0
; VI-NEXT: s_add_u32 s26, s26, 3
; VI-NEXT: s_addc_u32 s27, s27, 0
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7470,17 +7883,23 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v6i64_to_v6f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_3
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -7493,7 +7912,7 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s25, s25, 0
; GFX9-NEXT: s_add_u32 s26, s26, 3
; GFX9-NEXT: s_addc_u32 s27, s27, 0
-; GFX9-NEXT: .LBB39_3: ; %end
+; GFX9-NEXT: .LBB39_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7507,19 +7926,22 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_4:
-; GFX9-NEXT: s_branch .LBB39_2
;
; GFX11-LABEL: bitcast_v6i64_to_v6f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_3
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -7532,7 +7954,7 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s21, s21, 0
; GFX11-NEXT: s_add_u32 s22, s22, 3
; GFX11-NEXT: s_addc_u32 s23, s23, 0
-; GFX11-NEXT: .LBB39_3: ; %end
+; GFX11-NEXT: .LBB39_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -7540,8 +7962,6 @@ define inreg <6 x double> @bitcast_v6i64_to_v6f64_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_4:
-; GFX11-NEXT: s_branch .LBB39_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7733,7 +8153,7 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s45, s27, 16
; SI-NEXT: s_lshr_b32 s44, s25, 16
@@ -7747,8 +8167,28 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -7767,22 +8207,8 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: v_mov_b32_e32 v7, s23
@@ -7807,7 +8233,7 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v14, s8
; SI-NEXT: v_mov_b32_e32 v13, s6
; SI-NEXT: v_mov_b32_e32 v12, s4
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -7850,10 +8276,18 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -7861,9 +8295,7 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7886,10 +8318,18 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -7897,9 +8337,7 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7927,11 +8365,16 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
@@ -7939,8 +8382,6 @@ define inreg <24 x i16> @bitcast_v6f64_to_v24i16_scalar(<6 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8269,7 +8710,7 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s56, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -8307,8 +8748,17 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -8369,7 +8819,7 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s47, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -8395,18 +8845,23 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v24i16_to_v6f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s27, 3
; VI-NEXT: s_and_b32 s4, s27, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -8467,7 +8922,7 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8481,17 +8936,23 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v24i16_to_v6f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
@@ -8505,9 +8966,7 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8535,11 +8994,16 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
@@ -8553,8 +9017,6 @@ define inreg <6 x double> @bitcast_v24i16_to_v6f64_scalar(<24 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8756,7 +9218,7 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s45, s27, 16
; SI-NEXT: s_lshr_b32 s44, s25, 16
@@ -8770,8 +9232,28 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -8790,22 +9272,8 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v1
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: v_mov_b32_e32 v9, s25
; SI-NEXT: v_mov_b32_e32 v7, s23
@@ -8830,7 +9298,7 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v14, s8
; SI-NEXT: v_mov_b32_e32 v13, s6
; SI-NEXT: v_mov_b32_e32 v12, s4
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -8873,10 +9341,18 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -8884,9 +9360,7 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8909,10 +9383,18 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
@@ -8920,9 +9402,7 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8950,11 +9430,16 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[6:7], s[18:19], 1.0
@@ -8962,8 +9447,6 @@ define inreg <24 x half> @bitcast_v6f64_to_v24f16_scalar(<6 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -9340,7 +9823,7 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s56, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -9378,8 +9861,17 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s56
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -9476,11 +9968,8 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9497,7 +9986,7 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -9516,10 +10005,18 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s27, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -9582,9 +10079,7 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v12
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9607,10 +10102,18 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v11, s27, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
@@ -9625,9 +10128,7 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9655,11 +10156,16 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
@@ -9673,8 +10179,6 @@ define inreg <6 x double> @bitcast_v24f16_to_v6f64_scalar(<24 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -9903,7 +10407,7 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_4
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s27, 16
; SI-NEXT: s_lshr_b32 s41, s25, 16
@@ -9917,8 +10421,28 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_3
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s26, s26, 3
; SI-NEXT: s_addc_u32 s27, s27, 0
; SI-NEXT: s_add_u32 s24, s24, 3
@@ -9943,7 +10467,7 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: .LBB49_3: ; %end
+; SI-NEXT: .LBB49_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -9993,29 +10517,23 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s4
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB49_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB49_2
;
; VI-LABEL: bitcast_v6i64_to_v24i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_4
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_3
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s26, s26, 3
; VI-NEXT: s_addc_u32 s27, s27, 0
; VI-NEXT: s_add_u32 s24, s24, 3
@@ -10028,7 +10546,7 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB49_3: ; %end
+; VI-NEXT: .LBB49_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10042,17 +10560,23 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_4:
-; VI-NEXT: s_branch .LBB49_2
;
; GFX9-LABEL: bitcast_v6i64_to_v24i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_3
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s26, s26, 3
; GFX9-NEXT: s_addc_u32 s27, s27, 0
; GFX9-NEXT: s_add_u32 s24, s24, 3
@@ -10065,7 +10589,7 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB49_3: ; %end
+; GFX9-NEXT: .LBB49_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10079,19 +10603,22 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB49_4:
-; GFX9-NEXT: s_branch .LBB49_2
;
; GFX11-LABEL: bitcast_v6i64_to_v24i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_3
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB49_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s22, s22, 3
; GFX11-NEXT: s_addc_u32 s23, s23, 0
; GFX11-NEXT: s_add_u32 s20, s20, 3
@@ -10104,7 +10631,7 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB49_3: ; %end
+; GFX11-NEXT: .LBB49_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10113,8 +10640,6 @@ define inreg <24 x i16> @bitcast_v6i64_to_v24i16_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB49_4:
-; GFX11-NEXT: s_branch .LBB49_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10433,7 +10958,7 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s56, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -10471,8 +10996,17 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -10533,7 +11067,7 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s47, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10559,18 +11093,23 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v24i16_to_v6i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s27, 3
; VI-NEXT: s_and_b32 s4, s27, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -10631,7 +11170,7 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10645,17 +11184,23 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v24i16_to_v6i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
@@ -10669,9 +11214,7 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10699,11 +11242,16 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
@@ -10717,8 +11265,6 @@ define inreg <6 x i64> @bitcast_v24i16_to_v6i64_scalar(<24 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -10947,7 +11493,7 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s40, s27, 16
; SI-NEXT: s_lshr_b32 s41, s25, 16
@@ -10961,8 +11507,28 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[28:29], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[28:29], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s26, s26, 3
; SI-NEXT: s_addc_u32 s27, s27, 0
; SI-NEXT: s_add_u32 s24, s24, 3
@@ -10987,7 +11553,7 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[10:11], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s14, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -11037,29 +11603,23 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v10, s4
; SI-NEXT: v_mov_b32_e32 v11, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v6i64_to_v24f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s26, s26, 3
; VI-NEXT: s_addc_u32 s27, s27, 0
; VI-NEXT: s_add_u32 s24, s24, 3
@@ -11072,7 +11632,7 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11086,17 +11646,23 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v6i64_to_v24f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s26, s26, 3
; GFX9-NEXT: s_addc_u32 s27, s27, 0
; GFX9-NEXT: s_add_u32 s24, s24, 3
@@ -11109,7 +11675,7 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11123,19 +11689,22 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s26
; GFX9-NEXT: v_mov_b32_e32 v11, s27
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v6i64_to_v24f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s22, s22, 3
; GFX11-NEXT: s_addc_u32 s23, s23, 0
; GFX11-NEXT: s_add_u32 s20, s20, 3
@@ -11148,7 +11717,7 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11157,8 +11726,6 @@ define inreg <24 x half> @bitcast_v6i64_to_v24f16_scalar(<6 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11525,7 +12092,7 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s29, s17, 16
; SI-NEXT: s_lshr_b32 s56, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -11563,8 +12130,17 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s47, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s56
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s29
@@ -11661,11 +12237,8 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11682,7 +12255,7 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -11701,10 +12274,18 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s27, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -11767,9 +12348,7 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v12
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11792,10 +12371,18 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v11, s27, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
@@ -11810,9 +12397,7 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11840,11 +12425,16 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
@@ -11858,8 +12448,6 @@ define inreg <6 x i64> @bitcast_v24f16_to_v6i64_scalar(<24 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -12231,7 +12819,7 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s60, s17, 16
; SI-NEXT: s_lshr_b32 s74, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s60, 16
@@ -12281,8 +12869,28 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s12, s12, s56
; SI-NEXT: s_mov_b32 s13, s57
; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s29, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s29, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s79, 16
@@ -12355,7 +12963,7 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s63, s5, 16
; SI-NEXT: s_lshr_b32 s72, s15, 16
; SI-NEXT: s_lshr_b32 s73, s13, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s16, s28, 16
; SI-NEXT: s_or_b32 s10, s10, s16
@@ -12405,29 +13013,23 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v10, s12
; SI-NEXT: v_mov_b32_e32 v11, s13
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v24i16_to_v24f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
@@ -12488,7 +13090,7 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12502,17 +13104,23 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v10, s26
; VI-NEXT: v_mov_b32_e32 v11, s27
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v24i16_to_v24f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v10, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v9, s25, 3 op_sel_hi:[1,0]
@@ -12526,9 +13134,7 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12556,11 +13162,16 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v10, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v9, s21, 3 op_sel_hi:[1,0]
@@ -12574,8 +13185,6 @@ define inreg <24 x half> @bitcast_v24i16_to_v24f16_scalar(<24 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: s_branch .LBB57_2
; GFX11-NEXT: .LBB57_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -12902,10 +13511,18 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s40, s16, 16
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v1, s29
; SI-NEXT: v_cvt_f32_f16_e32 v2, s15
; SI-NEXT: v_cvt_f32_f16_e32 v3, s14
@@ -13008,10 +13625,8 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[16:17], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[14:15], v[8:9], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[10:11], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v24, s13
; SI-NEXT: v_mov_b32_e32 v28, s10
; SI-NEXT: v_mov_b32_e32 v25, s12
@@ -13036,7 +13651,7 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v16, s14
; SI-NEXT: v_mov_b32_e32 v14, s9
; SI-NEXT: v_mov_b32_e32 v12, s8
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v22
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v34
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -13079,10 +13694,18 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s26, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s5
@@ -13145,9 +13768,7 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v0, v1
; VI-NEXT: v_or_b32_e32 v0, v12, v13
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13170,10 +13791,18 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v11, s27, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v10, s26, v0 op_sel_hi:[1,0]
@@ -13188,9 +13817,7 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13218,11 +13845,16 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v10, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s21 op_sel_hi:[0,1]
@@ -13236,8 +13868,6 @@ define inreg <24 x i16> @bitcast_v24f16_to_v24i16_scalar(<24 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
index 04ccab7cc4b61..910198216abe7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.448bit.ll
@@ -138,10 +138,18 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s27, s27, 3
@@ -156,7 +164,7 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -172,18 +180,24 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v14i32_to_v14f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s27, s27, 3
@@ -198,7 +212,7 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -214,18 +228,24 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v14i32_to_v14f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s29, s29, 3
; GFX9-NEXT: s_add_i32 s28, s28, 3
; GFX9-NEXT: s_add_i32 s27, s27, 3
@@ -240,7 +260,7 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -256,19 +276,22 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v14i32_to_v14f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s25, s25, 3
; GFX11-NEXT: s_add_i32 s24, s24, 3
; GFX11-NEXT: s_add_i32 s23, s23, 3
@@ -283,7 +306,7 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -293,8 +316,6 @@ define inreg <14 x float> @bitcast_v14i32_to_v14f32_scalar(<14 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -437,10 +458,18 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -456,9 +485,7 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -482,10 +509,18 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
; VI-NEXT: v_add_f32_e64 v12, s28, 1.0
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -501,9 +536,7 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -527,10 +560,18 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX9-NEXT: v_add_f32_e64 v12, s28, 1.0
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -546,9 +587,7 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -576,11 +615,16 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v12, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
@@ -596,8 +640,6 @@ define inreg <14 x i32> @bitcast_v14f32_to_v14i32_scalar(<14 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -757,10 +799,18 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s27, s27, 3
@@ -775,7 +825,7 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -791,18 +841,24 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v14i32_to_v7i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s27, s27, 3
@@ -817,7 +873,7 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -833,18 +889,24 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v14i32_to_v7i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s29, s29, 3
; GFX9-NEXT: s_add_i32 s28, s28, 3
; GFX9-NEXT: s_add_i32 s27, s27, 3
@@ -859,7 +921,7 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -875,19 +937,22 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v14i32_to_v7i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s25, s25, 3
; GFX11-NEXT: s_add_i32 s24, s24, 3
; GFX11-NEXT: s_add_i32 s23, s23, 3
@@ -902,7 +967,7 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -912,8 +977,6 @@ define inreg <7 x i64> @bitcast_v14i32_to_v7i64_scalar(<14 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1067,10 +1130,18 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
; SI-NEXT: s_add_u32 s26, s26, 3
@@ -1085,7 +1156,7 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1101,18 +1172,24 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v7i64_to_v14i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s28, s28, 3
; VI-NEXT: s_addc_u32 s29, s29, 0
; VI-NEXT: s_add_u32 s26, s26, 3
@@ -1127,7 +1204,7 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1143,18 +1220,24 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v7i64_to_v14i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s28, s28, 3
; GFX9-NEXT: s_addc_u32 s29, s29, 0
; GFX9-NEXT: s_add_u32 s26, s26, 3
@@ -1169,7 +1252,7 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1185,19 +1268,22 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v7i64_to_v14i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s24, s24, 3
; GFX11-NEXT: s_addc_u32 s25, s25, 0
; GFX11-NEXT: s_add_u32 s22, s22, 3
@@ -1212,7 +1298,7 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1222,8 +1308,6 @@ define inreg <14 x i32> @bitcast_v7i64_to_v14i32_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1373,10 +1457,18 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s27, s27, 3
@@ -1391,7 +1483,7 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1407,18 +1499,24 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v14i32_to_v7f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s27, s27, 3
@@ -1433,7 +1531,7 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1449,18 +1547,24 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v14i32_to_v7f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s29, s29, 3
; GFX9-NEXT: s_add_i32 s28, s28, 3
; GFX9-NEXT: s_add_i32 s27, s27, 3
@@ -1475,7 +1579,7 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1491,19 +1595,22 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v14i32_to_v7f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s25, s25, 3
; GFX11-NEXT: s_add_i32 s24, s24, 3
; GFX11-NEXT: s_add_i32 s23, s23, 3
@@ -1518,7 +1625,7 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1528,8 +1635,6 @@ define inreg <7 x double> @bitcast_v14i32_to_v7f64_scalar(<14 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1651,10 +1756,18 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -1663,9 +1776,7 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1689,10 +1800,18 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -1701,9 +1820,7 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1727,10 +1844,18 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -1739,9 +1864,7 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1769,11 +1892,16 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
@@ -1782,8 +1910,6 @@ define inreg <14 x i32> @bitcast_v7f64_to_v14i32_scalar(<7 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2030,7 +2156,7 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s44, s29, 16
; SI-NEXT: s_lshr_b32 s45, s27, 16
@@ -2046,8 +2172,30 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -2076,7 +2224,7 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s56, s21, 16
; SI-NEXT: s_lshr_b32 s57, s19, 16
; SI-NEXT: s_lshr_b32 s58, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s40, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -2134,32 +2282,24 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v12, s4
; SI-NEXT: v_mov_b32_e32 v13, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v14i32_to_v28i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s27, s27, 3
@@ -2174,7 +2314,7 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2190,18 +2330,24 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v14i32_to_v28i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s29, s29, 3
; GFX9-NEXT: s_add_i32 s28, s28, 3
; GFX9-NEXT: s_add_i32 s27, s27, 3
@@ -2216,7 +2362,7 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2232,19 +2378,22 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v14i32_to_v28i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s25, s25, 3
; GFX11-NEXT: s_add_i32 s24, s24, 3
; GFX11-NEXT: s_add_i32 s23, s23, 3
@@ -2259,7 +2408,7 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2269,8 +2418,6 @@ define inreg <28 x i16> @bitcast_v14i32_to_v28i16_scalar(<14 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2624,7 +2771,7 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -2668,8 +2815,17 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -2740,7 +2896,7 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2768,19 +2924,24 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v28i16_to_v14i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s29, 3
; VI-NEXT: s_and_b32 s4, s29, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -2851,7 +3012,7 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2867,18 +3028,24 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v28i16_to_v14i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v12, s28, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
@@ -2894,9 +3061,7 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2924,11 +3089,16 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v12, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
@@ -2944,8 +3114,6 @@ define inreg <14 x i32> @bitcast_v28i16_to_v14i32_scalar(<28 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3192,7 +3360,7 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s44, s29, 16
; SI-NEXT: s_lshr_b32 s45, s27, 16
@@ -3208,8 +3376,30 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3238,7 +3428,7 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s56, s21, 16
; SI-NEXT: s_lshr_b32 s57, s19, 16
; SI-NEXT: s_lshr_b32 s58, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s40, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -3296,32 +3486,24 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s4
; SI-NEXT: v_mov_b32_e32 v13, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v14i32_to_v28f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s27, s27, 3
@@ -3336,7 +3518,7 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3352,18 +3534,24 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v14i32_to_v28f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s29, s29, 3
; GFX9-NEXT: s_add_i32 s28, s28, 3
; GFX9-NEXT: s_add_i32 s27, s27, 3
@@ -3378,7 +3566,7 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3394,19 +3582,22 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v14i32_to_v28f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s25, s25, 3
; GFX11-NEXT: s_add_i32 s24, s24, 3
; GFX11-NEXT: s_add_i32 s23, s23, 3
@@ -3421,7 +3612,7 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3431,8 +3622,6 @@ define inreg <28 x half> @bitcast_v14i32_to_v28f16_scalar(<14 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3842,7 +4031,7 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -3886,8 +4075,17 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -4000,11 +4198,8 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -4021,7 +4216,7 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB19_5: ; %end
+; SI-NEXT: .LBB19_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -4041,10 +4236,18 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s29, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -4117,9 +4320,7 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4143,10 +4344,18 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v13, s29, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v12, s28, v0 op_sel_hi:[1,0]
@@ -4163,9 +4372,7 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4193,11 +4400,16 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v12, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
@@ -4213,8 +4425,6 @@ define inreg <14 x i32> @bitcast_v28f16_to_v14i32_scalar(<28 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4367,10 +4577,18 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -4386,9 +4604,7 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4412,10 +4628,18 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
; VI-NEXT: v_add_f32_e64 v12, s28, 1.0
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -4431,9 +4655,7 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4457,10 +4679,18 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX9-NEXT: v_add_f32_e64 v12, s28, 1.0
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -4476,9 +4706,7 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4506,11 +4734,16 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v12, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
@@ -4526,8 +4759,6 @@ define inreg <7 x i64> @bitcast_v14f32_to_v7i64_scalar(<14 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -4691,10 +4922,18 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
; SI-NEXT: s_add_u32 s26, s26, 3
@@ -4709,7 +4948,7 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4725,18 +4964,24 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v7i64_to_v14f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s28, s28, 3
; VI-NEXT: s_addc_u32 s29, s29, 0
; VI-NEXT: s_add_u32 s26, s26, 3
@@ -4751,7 +4996,7 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4767,18 +5012,24 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v7i64_to_v14f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s28, s28, 3
; GFX9-NEXT: s_addc_u32 s29, s29, 0
; GFX9-NEXT: s_add_u32 s26, s26, 3
@@ -4793,7 +5044,7 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4809,19 +5060,22 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v7i64_to_v14f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s24, s24, 3
; GFX11-NEXT: s_addc_u32 s25, s25, 0
; GFX11-NEXT: s_add_u32 s22, s22, 3
@@ -4836,7 +5090,7 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -4846,8 +5100,6 @@ define inreg <14 x float> @bitcast_v7i64_to_v14f32_scalar(<7 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4990,10 +5242,18 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5009,9 +5269,7 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5035,10 +5293,18 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
; VI-NEXT: v_add_f32_e64 v12, s28, 1.0
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5054,9 +5320,7 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5080,10 +5344,18 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX9-NEXT: v_add_f32_e64 v12, s28, 1.0
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5099,9 +5371,7 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5129,11 +5399,16 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v12, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
@@ -5149,8 +5424,6 @@ define inreg <7 x double> @bitcast_v14f32_to_v7f64_scalar(<14 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -5282,10 +5555,18 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -5294,9 +5575,7 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5320,10 +5599,18 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -5332,9 +5619,7 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5358,10 +5643,18 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -5370,9 +5663,7 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5400,11 +5691,16 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
@@ -5413,8 +5709,6 @@ define inreg <14 x float> @bitcast_v7f64_to_v14f32_scalar(<7 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -5654,7 +5948,7 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s58, s29, 16
; SI-NEXT: s_lshr_b32 s57, s27, 16
@@ -5670,8 +5964,30 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5700,24 +6016,8 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -5746,7 +6046,7 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s12
; SI-NEXT: v_mov_b32_e32 v19, s14
; SI-NEXT: v_mov_b32_e32 v20, s40
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
@@ -5796,10 +6096,18 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
; VI-NEXT: v_add_f32_e64 v12, s28, 1.0
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5815,9 +6123,7 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5841,10 +6147,18 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX9-NEXT: v_add_f32_e64 v12, s28, 1.0
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -5860,9 +6174,7 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5890,11 +6202,16 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v12, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
@@ -5910,8 +6227,6 @@ define inreg <28 x i16> @bitcast_v14f32_to_v28i16_scalar(<14 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -6275,7 +6590,7 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -6319,8 +6634,17 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -6391,7 +6715,7 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -6419,19 +6743,24 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v28i16_to_v14f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s29, 3
; VI-NEXT: s_and_b32 s4, s29, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -6502,7 +6831,7 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6518,18 +6847,24 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v28i16_to_v14f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v12, s28, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
@@ -6545,9 +6880,7 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6575,11 +6908,16 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v12, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
@@ -6595,8 +6933,6 @@ define inreg <14 x float> @bitcast_v28i16_to_v14f32_scalar(<28 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -6836,7 +7172,7 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s58, s29, 16
; SI-NEXT: s_lshr_b32 s57, s27, 16
@@ -6852,8 +7188,30 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
; SI-NEXT: v_add_f32_e64 v12, s28, 1.0
; SI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -6882,24 +7240,8 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6928,7 +7270,7 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s12
; SI-NEXT: v_mov_b32_e32 v19, s14
; SI-NEXT: v_mov_b32_e32 v20, s40
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
@@ -6978,10 +7320,18 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
; VI-NEXT: v_add_f32_e64 v12, s28, 1.0
; VI-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -6997,9 +7347,7 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7023,10 +7371,18 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX9-NEXT: v_add_f32_e64 v12, s28, 1.0
; GFX9-NEXT: v_add_f32_e64 v11, s27, 1.0
@@ -7042,9 +7398,7 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7072,11 +7426,16 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v12, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v11, s23, 1.0
@@ -7092,8 +7451,6 @@ define inreg <28 x half> @bitcast_v14f32_to_v28f16_scalar(<14 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -7513,7 +7870,7 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -7557,8 +7914,17 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -7671,11 +8037,8 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -7692,7 +8055,7 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: .LBB35_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -7712,10 +8075,18 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s29, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -7788,9 +8159,7 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7814,10 +8183,18 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v13, s29, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v12, s28, v0 op_sel_hi:[1,0]
@@ -7834,9 +8211,7 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7864,11 +8239,16 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v12, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
@@ -7884,8 +8264,6 @@ define inreg <14 x float> @bitcast_v28f16_to_v14f32_scalar(<28 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8049,10 +8427,18 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -8067,7 +8453,7 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s27, s27, 0
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8083,18 +8469,24 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s28
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v7i64_to_v7f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -8109,7 +8501,7 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s27, s27, 0
; VI-NEXT: s_add_u32 s28, s28, 3
; VI-NEXT: s_addc_u32 s29, s29, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8125,18 +8517,24 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v7i64_to_v7f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -8151,7 +8549,7 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s27, s27, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
; GFX9-NEXT: s_addc_u32 s29, s29, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8167,19 +8565,22 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v7i64_to_v7f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -8194,7 +8595,7 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s23, s23, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
; GFX11-NEXT: s_addc_u32 s25, s25, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -8203,8 +8604,6 @@ define inreg <7 x double> @bitcast_v7i64_to_v7f64_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8326,10 +8725,18 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -8338,9 +8745,7 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8364,10 +8769,18 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -8376,9 +8789,7 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8402,10 +8813,18 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -8414,9 +8833,7 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8444,11 +8861,16 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
@@ -8457,8 +8879,6 @@ define inreg <7 x i64> @bitcast_v7f64_to_v7i64_scalar(<7 x double> inreg %a, i32
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8709,7 +9129,7 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s44, s29, 16
; SI-NEXT: s_lshr_b32 s45, s27, 16
@@ -8725,8 +9145,30 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
; SI-NEXT: s_add_u32 s26, s26, 3
@@ -8755,7 +9197,7 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s40, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -8813,32 +9255,24 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s4
; SI-NEXT: v_mov_b32_e32 v13, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v7i64_to_v28i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s28, s28, 3
; VI-NEXT: s_addc_u32 s29, s29, 0
; VI-NEXT: s_add_u32 s26, s26, 3
@@ -8853,7 +9287,7 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8869,18 +9303,24 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v7i64_to_v28i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s28, s28, 3
; GFX9-NEXT: s_addc_u32 s29, s29, 0
; GFX9-NEXT: s_add_u32 s26, s26, 3
@@ -8895,7 +9335,7 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8911,19 +9351,22 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v7i64_to_v28i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s24, s24, 3
; GFX11-NEXT: s_addc_u32 s25, s25, 0
; GFX11-NEXT: s_add_u32 s22, s22, 3
@@ -8938,7 +9381,7 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8948,8 +9391,6 @@ define inreg <28 x i16> @bitcast_v7i64_to_v28i16_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9303,7 +9744,7 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -9347,8 +9788,17 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -9419,7 +9869,7 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9447,19 +9897,24 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v28i16_to_v7i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s29, 3
; VI-NEXT: s_and_b32 s4, s29, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -9530,7 +9985,7 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9546,18 +10001,24 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v28i16_to_v7i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v12, s28, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
@@ -9573,9 +10034,7 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9603,11 +10062,16 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v12, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
@@ -9623,8 +10087,6 @@ define inreg <7 x i64> @bitcast_v28i16_to_v7i64_scalar(<28 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -9875,7 +10337,7 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s44, s29, 16
; SI-NEXT: s_lshr_b32 s45, s27, 16
@@ -9891,8 +10353,30 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s28, s28, 3
; SI-NEXT: s_addc_u32 s29, s29, 0
; SI-NEXT: s_add_u32 s26, s26, 3
@@ -9921,7 +10405,7 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s7, s40, 16
; SI-NEXT: s_or_b32 s5, s5, s7
@@ -9979,32 +10463,24 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v12, s4
; SI-NEXT: v_mov_b32_e32 v13, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v7i64_to_v28f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s28, s28, 3
; VI-NEXT: s_addc_u32 s29, s29, 0
; VI-NEXT: s_add_u32 s26, s26, 3
@@ -10019,7 +10495,7 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10035,18 +10511,24 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v7i64_to_v28f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s28, s28, 3
; GFX9-NEXT: s_addc_u32 s29, s29, 0
; GFX9-NEXT: s_add_u32 s26, s26, 3
@@ -10061,7 +10543,7 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10077,19 +10559,22 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v12, s28
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v7i64_to_v28f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s24, s24, 3
; GFX11-NEXT: s_addc_u32 s25, s25, 0
; GFX11-NEXT: s_add_u32 s22, s22, 3
@@ -10104,7 +10589,7 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10114,8 +10599,6 @@ define inreg <28 x half> @bitcast_v7i64_to_v28f16_scalar(<7 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v10, s22 :: v_dual_mov_b32 v11, s23
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10525,7 +11008,7 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -10569,8 +11052,17 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -10683,11 +11175,8 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10704,7 +11193,7 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -10724,10 +11213,18 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s29, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -10800,9 +11297,7 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10826,10 +11321,18 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v13, s29, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v12, s28, v0 op_sel_hi:[1,0]
@@ -10846,9 +11349,7 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10876,11 +11377,16 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v12, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
@@ -10896,8 +11402,6 @@ define inreg <7 x i64> @bitcast_v28f16_to_v7i64_scalar(<28 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -11116,7 +11620,7 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s58, s29, 16
; SI-NEXT: s_lshr_b32 s57, s27, 16
@@ -11132,8 +11636,30 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -11155,24 +11681,8 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: v_mov_b32_e32 v9, s25
@@ -11201,7 +11711,7 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v16, s8
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: v_mov_b32_e32 v14, s4
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
@@ -11251,10 +11761,18 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -11263,9 +11781,7 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11289,10 +11805,18 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -11301,9 +11825,7 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11331,11 +11853,16 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB49_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
@@ -11344,8 +11871,6 @@ define inreg <28 x i16> @bitcast_v7f64_to_v28i16_scalar(<7 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: s_branch .LBB49_2
; GFX11-NEXT: .LBB49_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -11709,7 +12234,7 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -11753,8 +12278,17 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -11825,7 +12359,7 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11853,19 +12387,24 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v28i16_to_v7f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s29, 3
; VI-NEXT: s_and_b32 s4, s29, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -11936,7 +12475,7 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11952,18 +12491,24 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v28i16_to_v7f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v12, s28, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
@@ -11979,9 +12524,7 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12009,11 +12552,16 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v12, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
@@ -12029,8 +12577,6 @@ define inreg <7 x double> @bitcast_v28i16_to_v7f64_scalar(<28 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -12249,7 +12795,7 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s58, s29, 16
; SI-NEXT: s_lshr_b32 s57, s27, 16
@@ -12265,8 +12811,30 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[42:43], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[42:43], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -12288,24 +12856,8 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v11, s27
; SI-NEXT: v_mov_b32_e32 v9, s25
@@ -12334,7 +12886,7 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v16, s8
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: v_mov_b32_e32 v14, s4
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
@@ -12384,10 +12936,18 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -12396,9 +12956,7 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12422,10 +12980,18 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
@@ -12434,9 +13000,7 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12464,11 +13028,16 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
; GFX11-NEXT: v_add_f64 v[8:9], s[20:21], 1.0
@@ -12477,8 +13046,6 @@ define inreg <28 x half> @bitcast_v7f64_to_v28f16_scalar(<7 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_3:
-; GFX11-NEXT: s_branch .LBB53_2
; GFX11-NEXT: .LBB53_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -12898,7 +13465,7 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s59, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s59, 16
@@ -12942,8 +13509,17 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s58
@@ -13056,11 +13632,8 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v12, v13, v12
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v14
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -13077,7 +13650,7 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v16, 7
; SI-NEXT: v_readlane_b32 s50, v16, 6
; SI-NEXT: v_readlane_b32 s49, v16, 5
@@ -13097,10 +13670,18 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s29, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -13173,9 +13754,7 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v14
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13199,10 +13778,18 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v13, s29, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v12, s28, v0 op_sel_hi:[1,0]
@@ -13219,9 +13806,7 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13249,11 +13834,16 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v12, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
@@ -13269,8 +13859,6 @@ define inreg <7 x double> @bitcast_v28f16_to_v7f64_scalar(<28 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -13708,7 +14296,7 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s89, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s74, 16
@@ -13766,8 +14354,30 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s14, s14, s62
; SI-NEXT: s_mov_b32 s15, s63
; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[72:73], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[72:73], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[72:73], s[72:73], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -13852,7 +14462,7 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s78, s5, 16
; SI-NEXT: s_lshr_b32 s79, s41, 16
; SI-NEXT: s_lshr_b32 s88, s15, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s16, s42, 16
; SI-NEXT: s_or_b32 s12, s12, s16
@@ -13910,32 +14520,24 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v12, s14
; SI-NEXT: v_mov_b32_e32 v13, s15
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v28i16_to_v28f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
@@ -14006,7 +14608,7 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14022,18 +14624,24 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v12, s28
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v28i16_to_v28f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v12, s28, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v11, s27, 3 op_sel_hi:[1,0]
@@ -14049,9 +14657,7 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14079,11 +14685,16 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v12, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v11, s23, 3 op_sel_hi:[1,0]
@@ -14099,8 +14710,6 @@ define inreg <28 x half> @bitcast_v28i16_to_v28f16_scalar(<28 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: s_branch .LBB57_2
; GFX11-NEXT: .LBB57_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -14465,10 +15074,18 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s43, s16, 16
; SI-NEXT: v_readfirstlane_b32 s4, v0
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v1, s42
; SI-NEXT: v_cvt_f32_f16_e32 v2, s16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s41
@@ -14588,10 +15205,8 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[18:19], v[8:9], 16
; SI-NEXT: v_lshr_b64 v[16:17], v[10:11], 16
; SI-NEXT: v_lshr_b64 v[14:15], v[12:13], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v28, s15
; SI-NEXT: v_mov_b32_e32 v33, s10
; SI-NEXT: v_mov_b32_e32 v29, s11
@@ -14620,7 +15235,7 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v18, s14
; SI-NEXT: v_mov_b32_e32 v16, s13
; SI-NEXT: v_mov_b32_e32 v14, s12
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v26
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v49
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -14670,10 +15285,18 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s28, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v2, s5
@@ -14746,9 +15369,7 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v0, v1
; VI-NEXT: v_or_b32_e32 v0, v14, v15
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14772,10 +15393,18 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v13, s29, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v12, s28, v0 op_sel_hi:[1,0]
@@ -14792,9 +15421,7 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14822,11 +15449,16 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s26, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v12, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v11, 0x200, s23 op_sel_hi:[0,1]
@@ -14842,8 +15474,6 @@ define inreg <28 x i16> @bitcast_v28f16_to_v28i16_scalar(<28 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 62cd5098d32f2..5c9706d96e4ed 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -253,15 +253,26 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v9
; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
@@ -273,25 +284,28 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v3bf16_to_v3f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_3
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_4
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -322,9 +336,7 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, 0x7fc00000
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_3:
-; VI-NEXT: s_branch .LBB1_2
-; VI-NEXT: .LBB1_4:
+; VI-NEXT: .LBB1_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -333,10 +345,18 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_4
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -370,9 +390,7 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_3:
-; GFX9-NEXT: s_branch .LBB1_2
-; GFX9-NEXT: .LBB1_4:
+; GFX9-NEXT: .LBB1_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -382,11 +400,16 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB1_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB1_4
-; GFX11-TRUE16-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB1_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -424,8 +447,6 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB1_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB1_2
; GFX11-TRUE16-NEXT: .LBB1_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -435,11 +456,16 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB1_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB1_4
-; GFX11-FAKE16-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB1_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s0, 16
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -478,8 +504,6 @@ define inreg <3 x half> @bitcast_v3bf16_to_v3f16_scalar(<3 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, 0x7fc0, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB1_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB1_2
; GFX11-FAKE16-NEXT: .LBB1_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -614,13 +638,24 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s5, s16, 16
; SI-NEXT: s_lshl_b32 s8, s4, 16
; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s17
; SI-NEXT: v_cvt_f32_f16_e32 v1, s4
; SI-NEXT: v_cvt_f32_f16_e32 v2, s16
@@ -633,17 +668,12 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v2, s9
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -656,10 +686,18 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v3, s4
@@ -669,9 +707,7 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: v_or_b32_e32 v1, 0x7e000000, v1
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -680,17 +716,23 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -700,16 +742,19 @@ define inreg <3 x bfloat> @bitcast_v3f16_to_v3bf16_scalar(<3 x half> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -969,13 +1014,24 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
@@ -987,25 +1043,28 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_lshr_b64 v[3:4], v[0:1], 16
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $vgpr2
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v3bf16_to_v3i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_3
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_4
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v1
@@ -1036,9 +1095,7 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v1, 0x7fc00000
; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_3:
-; VI-NEXT: s_branch .LBB5_2
-; VI-NEXT: .LBB5_4:
+; VI-NEXT: .LBB5_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1047,10 +1104,18 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_4
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_lshl_b32 s4, s17, 16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -1084,9 +1149,7 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_3:
-; GFX9-NEXT: s_branch .LBB5_2
-; GFX9-NEXT: .LBB5_4:
+; GFX9-NEXT: .LBB5_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1096,11 +1159,16 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB5_4
-; GFX11-TRUE16-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB5_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s0, 16
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -1134,8 +1202,6 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB5_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB5_2
; GFX11-TRUE16-NEXT: .LBB5_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1145,11 +1211,16 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB5_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB5_4
-; GFX11-FAKE16-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB5_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s0, 16
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -1185,8 +1256,6 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v2, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB5_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB5_2
; GFX11-FAKE16-NEXT: .LBB5_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -1309,13 +1378,24 @@ define inreg <3 x bfloat> @bitcast_v3i16_to_v3bf16_scalar(<3 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s5, s16, 16
; SI-NEXT: s_lshl_b32 s8, s4, 16
; SI-NEXT: s_lshl_b32 s9, s17, 16
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s5, s16, 3
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -1325,7 +1405,7 @@ define inreg <3 x bfloat> @bitcast_v3i16_to_v3bf16_scalar(<3 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s4, 16
; SI-NEXT: s_add_i32 s9, s6, 0x30000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s5
@@ -1333,47 +1413,54 @@ define inreg <3 x bfloat> @bitcast_v3i16_to_v3bf16_scalar(<3 x i16> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v3i16_to_v3bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v3i16_to_v3bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_3:
-; GFX9-NEXT: s_branch .LBB7_2
-; GFX9-NEXT: .LBB7_4:
+; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1383,16 +1470,19 @@ define inreg <3 x bfloat> @bitcast_v3i16_to_v3bf16_scalar(<3 x i16> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_4
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_3:
-; GFX11-NEXT: s_branch .LBB7_2
; GFX11-NEXT: .LBB7_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1513,10 +1603,18 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_3
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_4
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s4
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s17
@@ -1529,14 +1627,12 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_or_b32_e32 v0, v4, v0
-; SI-NEXT: s_branch .LBB9_5
-; SI-NEXT: .LBB9_3:
-; SI-NEXT: s_branch .LBB9_2
-; SI-NEXT: .LBB9_4:
+; SI-NEXT: s_branch .LBB9_6
+; SI-NEXT: .LBB9_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: .LBB9_5: ; %end
+; SI-NEXT: .LBB9_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -1546,10 +1642,18 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_3
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_4
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v3, s4
@@ -1559,9 +1663,7 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: v_or_b32_e32 v1, 0x7e000000, v1
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_3:
-; VI-NEXT: s_branch .LBB9_2
-; VI-NEXT: .LBB9_4:
+; VI-NEXT: .LBB9_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1570,17 +1672,23 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_4
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_3:
-; GFX9-NEXT: s_branch .LBB9_2
-; GFX9-NEXT: .LBB9_4:
+; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1590,16 +1698,19 @@ define inreg <3 x i16> @bitcast_v3f16_to_v3i16_scalar(<3 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_4
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_3:
-; GFX11-NEXT: s_branch .LBB9_2
; GFX11-NEXT: .LBB9_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1726,7 +1837,7 @@ define inreg <3 x half> @bitcast_v3i16_to_v3f16_scalar(<3 x i16> inreg %a, i32 i
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_4
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s8, s6, 16
@@ -1734,8 +1845,19 @@ define inreg <3 x half> @bitcast_v3i16_to_v3f16_scalar(<3 x i16> inreg %a, i32 i
; SI-NEXT: s_or_b32 s4, s4, s8
; SI-NEXT: s_and_b32 s7, s17, 0xffff
; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
-; SI-NEXT: s_cbranch_execnz .LBB11_3
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s4, s16, 3
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
@@ -1744,54 +1866,61 @@ define inreg <3 x half> @bitcast_v3i16_to_v3f16_scalar(<3 x i16> inreg %a, i32 i
; SI-NEXT: s_add_i32 s5, s17, 3
; SI-NEXT: s_and_b32 s7, s5, 0xffff
; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
-; SI-NEXT: .LBB11_3: ; %end
+; SI-NEXT: .LBB11_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s5, s8, 16
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB11_2
;
; VI-LABEL: bitcast_v3i16_to_v3f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_4
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_3
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB11_3: ; %end
+; VI-NEXT: .LBB11_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_4:
-; VI-NEXT: s_branch .LBB11_2
;
; GFX9-LABEL: bitcast_v3i16_to_v3f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1801,16 +1930,19 @@ define inreg <3 x half> @bitcast_v3i16_to_v3f16_scalar(<3 x i16> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 092528a4feb1e..575d778195f84 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -148,10 +148,18 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s29, s29, 3
@@ -168,7 +176,7 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -186,8 +194,6 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v16i32_to_v16f32_scalar:
; VI: ; %bb.0:
@@ -196,10 +202,18 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -216,7 +230,7 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -234,8 +248,6 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v16i32_to_v16f32_scalar:
; GFX9: ; %bb.0:
@@ -244,10 +256,18 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -264,7 +284,7 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -282,19 +302,22 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v16i32_to_v16f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -311,7 +334,7 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -322,8 +345,6 @@ define inreg <16 x float> @bitcast_v16i32_to_v16f32_scalar(<16 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -481,10 +502,18 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v15, s31, 1.0
; SI-NEXT: v_add_f32_e64 v14, s30, 1.0
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -501,10 +530,8 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -521,7 +548,7 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -542,10 +569,18 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -562,10 +597,8 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -582,7 +615,7 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -603,10 +636,18 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -623,10 +664,8 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -643,7 +682,7 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -661,11 +700,16 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -683,8 +727,6 @@ define inreg <16 x i32> @bitcast_v16f32_to_v16i32_scalar(<16 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -854,10 +896,18 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s29, s29, 3
@@ -874,7 +924,7 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -892,8 +942,6 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v16i32_to_v8i64_scalar:
; VI: ; %bb.0:
@@ -902,10 +950,18 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -922,7 +978,7 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -940,8 +996,6 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v16i32_to_v8i64_scalar:
; GFX9: ; %bb.0:
@@ -950,10 +1004,18 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -970,7 +1032,7 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -988,19 +1050,22 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v16i32_to_v8i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -1017,7 +1082,7 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1028,8 +1093,6 @@ define inreg <8 x i64> @bitcast_v16i32_to_v8i64_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1193,10 +1256,18 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s28, s28, 3
@@ -1213,7 +1284,7 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1231,8 +1302,6 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v8i64_to_v16i32_scalar:
; VI: ; %bb.0:
@@ -1241,10 +1310,18 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -1261,7 +1338,7 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1279,8 +1356,6 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v8i64_to_v16i32_scalar:
; GFX9: ; %bb.0:
@@ -1289,10 +1364,18 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -1309,7 +1392,7 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1327,19 +1410,22 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v8i64_to_v16i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -1356,7 +1442,7 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1367,8 +1453,6 @@ define inreg <16 x i32> @bitcast_v8i64_to_v16i32_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1528,10 +1612,18 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s29, s29, 3
@@ -1548,7 +1640,7 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1566,8 +1658,6 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v16i32_to_v8f64_scalar:
; VI: ; %bb.0:
@@ -1576,10 +1666,18 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -1596,7 +1694,7 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1614,8 +1712,6 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v16i32_to_v8f64_scalar:
; GFX9: ; %bb.0:
@@ -1624,10 +1720,18 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -1644,7 +1748,7 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1662,19 +1766,22 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v16i32_to_v8f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -1691,7 +1798,7 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1702,8 +1809,6 @@ define inreg <8 x double> @bitcast_v16i32_to_v8f64_scalar(<16 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1837,10 +1942,18 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -1849,10 +1962,8 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1869,7 +1980,7 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -1890,10 +2001,18 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -1902,10 +2021,8 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1922,7 +2039,7 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -1943,10 +2060,18 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -1955,10 +2080,8 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1975,7 +2098,7 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -1993,11 +2116,16 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -2007,8 +2135,6 @@ define inreg <16 x i32> @bitcast_v8f64_to_v16i32_scalar(<8 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -2277,7 +2403,7 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s5, 16
; SI-NEXT: s_lshr_b32 s57, s29, 16
@@ -2295,8 +2421,32 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -2329,7 +2479,7 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s61, s21, 16
; SI-NEXT: s_lshr_b32 s62, s19, 16
; SI-NEXT: s_lshr_b32 s63, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s7, s16, 0xffff
; SI-NEXT: s_lshl_b32 s9, s44, 16
; SI-NEXT: s_or_b32 s7, s7, s9
@@ -2395,24 +2545,6 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v16i32_to_v32i16_scalar:
; VI: ; %bb.0:
@@ -2421,10 +2553,18 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -2441,7 +2581,7 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2459,8 +2599,6 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v16i32_to_v32i16_scalar:
; GFX9: ; %bb.0:
@@ -2469,10 +2607,18 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -2489,7 +2635,7 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2507,19 +2653,22 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v16i32_to_v32i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -2536,7 +2685,7 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2547,8 +2696,6 @@ define inreg <32 x i16> @bitcast_v16i32_to_v32i16_scalar(<16 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2938,7 +3085,7 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s11, s12, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -2988,8 +3135,17 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s9, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -3070,7 +3226,7 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -3100,9 +3256,6 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v32i16_to_v16i32_scalar:
; VI: ; %bb.0:
@@ -3111,10 +3264,18 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s6, 3
; VI-NEXT: s_and_b32 s4, s6, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -3195,7 +3356,7 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3213,8 +3374,6 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v32i16_to_v16i32_scalar:
; GFX9: ; %bb.0:
@@ -3228,10 +3387,18 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -3248,10 +3415,8 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3268,7 +3433,7 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -3286,11 +3451,16 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -3308,8 +3478,6 @@ define inreg <16 x i32> @bitcast_v32i16_to_v16i32_scalar(<32 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -3578,7 +3746,7 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s5, 16
; SI-NEXT: s_lshr_b32 s57, s29, 16
@@ -3596,8 +3764,32 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3630,7 +3822,7 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s61, s21, 16
; SI-NEXT: s_lshr_b32 s62, s19, 16
; SI-NEXT: s_lshr_b32 s63, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s7, s16, 0xffff
; SI-NEXT: s_lshl_b32 s9, s44, 16
; SI-NEXT: s_or_b32 s7, s7, s9
@@ -3696,24 +3888,6 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v16i32_to_v32f16_scalar:
; VI: ; %bb.0:
@@ -3722,10 +3896,18 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -3742,7 +3924,7 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -3760,8 +3942,6 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v16i32_to_v32f16_scalar:
; GFX9: ; %bb.0:
@@ -3770,10 +3950,18 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -3790,7 +3978,7 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3808,19 +3996,22 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v16i32_to_v32f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -3837,7 +4028,7 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -3848,8 +4039,6 @@ define inreg <32 x half> @bitcast_v16i32_to_v32f16_scalar(<16 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4303,7 +4492,7 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s10, s8, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -4353,8 +4542,17 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s62
@@ -4483,11 +4681,8 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v14, v15, v14
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -4504,7 +4699,7 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB19_5: ; %end
+; SI-NEXT: .LBB19_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v18, 7
; SI-NEXT: v_readlane_b32 s50, v18, 6
; SI-NEXT: v_readlane_b32 s49, v18, 5
@@ -4531,10 +4726,18 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s31, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -4616,10 +4819,8 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v16, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4636,7 +4837,7 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB19_5: ; %end
+; VI-NEXT: .LBB19_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v17, 0
; VI-NEXT: v_readlane_b32 s31, v17, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -4657,10 +4858,18 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -4678,10 +4887,8 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4698,7 +4905,7 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -4716,11 +4923,16 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -4738,8 +4950,6 @@ define inreg <16 x i32> @bitcast_v32f16_to_v16i32_scalar(<32 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -5088,7 +5298,7 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; SI-NEXT: v_readfirstlane_b32 s78, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s79, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s78, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s78, 16
@@ -5122,8 +5332,48 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; SI-NEXT: s_lshl_b32 s74, s17, 16
; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s76, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -5172,7 +5422,7 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; SI-NEXT: s_lshl_b32 s74, s17, 16
; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s76, s16, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
@@ -5238,40 +5488,6 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_v16i32_to_v32bf16_scalar:
; VI: ; %bb.0:
@@ -5280,10 +5496,18 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s29, s29, 3
@@ -5300,7 +5524,7 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5318,8 +5542,6 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_v16i32_to_v32bf16_scalar:
; GFX9: ; %bb.0:
@@ -5328,10 +5550,18 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -5348,7 +5578,7 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5366,19 +5596,22 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_v16i32_to_v32bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s27, s27, 3
; GFX11-NEXT: s_add_i32 s26, s26, 3
; GFX11-NEXT: s_add_i32 s25, s25, 3
@@ -5395,7 +5628,7 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5406,8 +5639,6 @@ define inreg <32 x bfloat> @bitcast_v16i32_to_v32bf16_scalar(<16 x i32> inreg %a
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6920,7 +7151,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
@@ -6956,8 +7187,17 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v20, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
; SI-NEXT: v_mov_b32_e32 v16, v20
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -7054,7 +7294,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -7073,9 +7313,6 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v32bf16_to_v16i32_scalar:
; VI: ; %bb.0:
@@ -7089,10 +7326,18 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s30, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v16
@@ -7390,10 +7635,8 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_lshrrev_b64 v[16:17], 16, v[17:18]
; VI-NEXT: v_mov_b32_e32 v13, v15
; VI-NEXT: v_mov_b32_e32 v15, v16
-; VI-NEXT: s_branch .LBB23_5
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: s_branch .LBB23_6
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -7410,7 +7653,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB23_5: ; %end
+; VI-NEXT: .LBB23_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -7431,10 +7674,18 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s31, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -7741,10 +7992,8 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
-; GFX9-NEXT: s_branch .LBB23_5
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: s_branch .LBB23_6
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -7761,7 +8010,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB23_5: ; %end
+; GFX9-NEXT: .LBB23_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -7779,11 +8028,16 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-TRUE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s27, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -8093,8 +8347,6 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB23_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB23_2
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -8115,11 +8367,16 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-FAKE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s27, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s27, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -8448,8 +8705,6 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v16, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v20, 16, v17
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB23_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB23_2
; GFX11-FAKE16-NEXT: .LBB23_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -9986,7 +10241,7 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s5, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_cbranch_scc0 .LBB25_4
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 24
; SI-NEXT: s_lshr_b32 s39, s5, 16
@@ -10033,11 +10288,67 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB25_3
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -10099,15 +10410,15 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
-; SI-NEXT: .LBB25_3: ; %end
-; SI-NEXT: s_lshl_b32 s7, s36, 8
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: s_lshl_b32 s7, s34, 8
; SI-NEXT: s_and_b32 s9, s16, 0xff
; SI-NEXT: s_or_b32 s7, s9, s7
-; SI-NEXT: s_and_b32 s9, s34, 0xff
-; SI-NEXT: s_lshl_b32 s11, s30, 24
+; SI-NEXT: s_and_b32 s9, s30, 0xff
+; SI-NEXT: s_lshl_b32 s11, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_or_b32 s9, s11, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
@@ -10342,56 +10653,6 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB25_2
;
; VI-LABEL: bitcast_v16i32_to_v64i8_scalar:
; VI: ; %bb.0:
@@ -10415,15 +10676,13 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: v_writelane_b32 v4, s55, 13
; VI-NEXT: v_writelane_b32 v4, s64, 14
; VI-NEXT: v_writelane_b32 v4, s65, 15
-; VI-NEXT: v_writelane_b32 v4, s66, 16
-; VI-NEXT: v_writelane_b32 v4, s67, 17
-; VI-NEXT: v_writelane_b32 v4, s30, 18
-; VI-NEXT: v_writelane_b32 v4, s31, 19
+; VI-NEXT: v_writelane_b32 v4, s30, 16
+; VI-NEXT: v_writelane_b32 v4, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB25_4
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s56, s5, 24
; VI-NEXT: s_lshr_b32 s57, s5, 16
@@ -10460,11 +10719,11 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[26:27], 24
@@ -10473,8 +10732,64 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[44:45], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB25_3
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s19, s19, 3
@@ -10534,23 +10849,23 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
-; VI-NEXT: .LBB25_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
+; VI-NEXT: .LBB25_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s44
-; VI-NEXT: v_mov_b32_e32 v1, s67
-; VI-NEXT: v_perm_b32 v3, s66, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s65
+; VI-NEXT: v_perm_b32 v3, s64, v3, v2
; VI-NEXT: v_perm_b32 v1, s16, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
-; VI-NEXT: v_mov_b32_e32 v3, s55
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v1, s65
-; VI-NEXT: v_perm_b32 v3, s64, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s55
+; VI-NEXT: v_perm_b32 v3, vcc_hi, v3, v2
; VI-NEXT: v_perm_b32 v1, s17, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -10667,11 +10982,9 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: v_add_u32_e32 v0, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s30, v4, 18
+; VI-NEXT: v_readlane_b32 s30, v4, 16
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_readlane_b32 s31, v4, 19
-; VI-NEXT: v_readlane_b32 s67, v4, 17
-; VI-NEXT: v_readlane_b32 s66, v4, 16
+; VI-NEXT: v_readlane_b32 s31, v4, 17
; VI-NEXT: v_readlane_b32 s65, v4, 15
; VI-NEXT: v_readlane_b32 s64, v4, 14
; VI-NEXT: v_readlane_b32 s55, v4, 13
@@ -10693,56 +11006,6 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_4:
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB25_2
;
; GFX9-LABEL: bitcast_v16i32_to_v64i8_scalar:
; GFX9: ; %bb.0:
@@ -10762,15 +11025,13 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: v_writelane_b32 v4, s51, 9
; GFX9-NEXT: v_writelane_b32 v4, s52, 10
; GFX9-NEXT: v_writelane_b32 v4, s53, 11
-; GFX9-NEXT: v_writelane_b32 v4, s54, 12
-; GFX9-NEXT: v_writelane_b32 v4, s55, 13
-; GFX9-NEXT: v_writelane_b32 v4, s30, 14
-; GFX9-NEXT: v_writelane_b32 v4, s31, 15
+; GFX9-NEXT: v_writelane_b32 v4, s30, 12
+; GFX9-NEXT: v_writelane_b32 v4, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s56, s5, 24
; GFX9-NEXT: s_lshr_b32 s57, s5, 16
@@ -10796,22 +11057,22 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s94, s23, 8
; GFX9-NEXT: s_lshr_b32 s95, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s35, s21, 8
-; GFX9-NEXT: s_lshr_b32 s36, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s48, s19, 8
-; GFX9-NEXT: s_lshr_b32 s49, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s53, s17, 8
-; GFX9-NEXT: s_lshr_b32 s54, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s31, s21, 8
+; GFX9-NEXT: s_lshr_b32 s34, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s38, s19, 8
+; GFX9-NEXT: s_lshr_b32 s39, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s51, s17, 8
+; GFX9-NEXT: s_lshr_b32 s52, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[26:27], 24
@@ -10820,8 +11081,64 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[44:45], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB25_3
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_add_i32 s19, s19, 3
@@ -10870,68 +11187,68 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s94, s23, 8
; GFX9-NEXT: s_lshr_b32 s95, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s35, s21, 8
-; GFX9-NEXT: s_lshr_b32 s36, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s48, s19, 8
-; GFX9-NEXT: s_lshr_b32 s49, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s53, s17, 8
-; GFX9-NEXT: s_lshr_b32 s54, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
-; GFX9-NEXT: .LBB25_3: ; %end
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s31, s21, 8
+; GFX9-NEXT: s_lshr_b32 s34, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s38, s19, 8
+; GFX9-NEXT: s_lshr_b32 s39, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s51, s17, 8
+; GFX9-NEXT: s_lshr_b32 s52, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
+; GFX9-NEXT: .LBB25_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v3, s44
-; GFX9-NEXT: v_mov_b32_e32 v1, s55
-; GFX9-NEXT: v_perm_b32 v3, s54, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s53
+; GFX9-NEXT: v_perm_b32 v3, s52, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s16, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s51
+; GFX9-NEXT: v_mov_b32_e32 v3, s49
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; GFX9-NEXT: v_mov_b32_e32 v1, s53
-; GFX9-NEXT: v_perm_b32 v3, s52, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s51
+; GFX9-NEXT: v_perm_b32 v3, s50, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s17, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s42
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT: v_mov_b32_e32 v1, s50
-; GFX9-NEXT: v_perm_b32 v3, s49, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s48
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s38
+; GFX9-NEXT: v_mov_b32_e32 v3, s36
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
-; GFX9-NEXT: v_mov_b32_e32 v1, s48
-; GFX9-NEXT: v_perm_b32 v3, s39, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s38
+; GFX9-NEXT: v_perm_b32 v3, s37, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s19, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s40
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
-; GFX9-NEXT: v_mov_b32_e32 v1, s37
-; GFX9-NEXT: v_perm_b32 v3, s36, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s35
+; GFX9-NEXT: v_perm_b32 v3, s34, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s31
+; GFX9-NEXT: v_mov_b32_e32 v3, vcc_hi
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
-; GFX9-NEXT: v_mov_b32_e32 v1, s35
-; GFX9-NEXT: v_perm_b32 v3, s34, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s31
+; GFX9-NEXT: v_perm_b32 v3, s30, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s21, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_mov_b32_e32 v1, s30
+; GFX9-NEXT: v_mov_b32_e32 v1, vcc_lo
; GFX9-NEXT: v_perm_b32 v3, s95, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s22, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
@@ -10999,11 +11316,9 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v2, s57, v3, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_readlane_b32 s30, v4, 14
+; GFX9-NEXT: v_readlane_b32 s30, v4, 12
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
-; GFX9-NEXT: v_readlane_b32 s31, v4, 15
-; GFX9-NEXT: v_readlane_b32 s55, v4, 13
-; GFX9-NEXT: v_readlane_b32 s54, v4, 12
+; GFX9-NEXT: v_readlane_b32 s31, v4, 13
; GFX9-NEXT: v_readlane_b32 s53, v4, 11
; GFX9-NEXT: v_readlane_b32 s52, v4, 10
; GFX9-NEXT: v_readlane_b32 s51, v4, 9
@@ -11021,56 +11336,6 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_4:
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: s_branch .LBB25_2
;
; GFX11-LABEL: bitcast_v16i32_to_v64i8_scalar:
; GFX11: ; %bb.0:
@@ -11088,8 +11353,8 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: v_writelane_b32 v23, s30, 7
; GFX11-NEXT: v_writelane_b32 v23, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX11-NEXT: s_mov_b32 s48, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s27, 24
; GFX11-NEXT: s_lshr_b32 s44, s27, 16
@@ -11121,16 +11386,16 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s91, s17, 8
; GFX11-NEXT: s_lshr_b32 s95, s16, 16
; GFX11-NEXT: s_lshr_b32 s94, s16, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
-; GFX11-NEXT: s_lshr_b32 s30, s3, 16
-; GFX11-NEXT: s_lshr_b32 s31, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s35, s2, 8
-; GFX11-NEXT: s_lshr_b32 s36, s1, 24
-; GFX11-NEXT: s_lshr_b32 s37, s1, 16
-; GFX11-NEXT: s_lshr_b32 s38, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s48, s0, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s3, 24
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 16
+; GFX11-NEXT: s_lshr_b32 s30, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s34, s2, 8
+; GFX11-NEXT: s_lshr_b32 s35, s1, 24
+; GFX11-NEXT: s_lshr_b32 s36, s1, 16
+; GFX11-NEXT: s_lshr_b32 s37, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s39, s0, 8
; GFX11-NEXT: s_lshr_b64 s[4:5], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[8:9], s[22:23], 24
@@ -11139,172 +11404,21 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[40:41], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_3
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
-; GFX11-NEXT: s_add_i32 s1, s1, 3
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s3, s3, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s17, s17, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s19, s19, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s21, s21, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s23, s23, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s25, s25, 3
-; GFX11-NEXT: s_add_i32 s27, s27, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_lshr_b64 s[4:5], s[26:27], 24
-; GFX11-NEXT: s_lshr_b64 s[6:7], s[24:25], 24
-; GFX11-NEXT: s_lshr_b64 s[8:9], s[22:23], 24
-; GFX11-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
-; GFX11-NEXT: s_lshr_b64 s[12:13], s[18:19], 24
-; GFX11-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
-; GFX11-NEXT: s_lshr_b64 s[28:29], s[2:3], 24
-; GFX11-NEXT: s_lshr_b64 s[40:41], s[0:1], 24
-; GFX11-NEXT: s_lshr_b32 s42, s27, 24
-; GFX11-NEXT: s_lshr_b32 s44, s27, 16
-; GFX11-NEXT: s_lshr_b32 s43, s27, 8
-; GFX11-NEXT: s_lshr_b32 s46, s26, 16
-; GFX11-NEXT: s_lshr_b32 s45, s26, 8
-; GFX11-NEXT: s_lshr_b32 s56, s25, 24
-; GFX11-NEXT: s_lshr_b32 s57, s25, 16
-; GFX11-NEXT: s_lshr_b32 s47, s25, 8
-; GFX11-NEXT: s_lshr_b32 s59, s24, 16
-; GFX11-NEXT: s_lshr_b32 s58, s24, 8
-; GFX11-NEXT: s_lshr_b32 s61, s23, 24
-; GFX11-NEXT: s_lshr_b32 s62, s23, 16
-; GFX11-NEXT: s_lshr_b32 s60, s23, 8
-; GFX11-NEXT: s_lshr_b32 s72, s22, 16
-; GFX11-NEXT: s_lshr_b32 s63, s22, 8
-; GFX11-NEXT: s_lshr_b32 s74, s21, 24
-; GFX11-NEXT: s_lshr_b32 s75, s21, 16
-; GFX11-NEXT: s_lshr_b32 s73, s21, 8
-; GFX11-NEXT: s_lshr_b32 s77, s20, 16
-; GFX11-NEXT: s_lshr_b32 s76, s20, 8
-; GFX11-NEXT: s_lshr_b32 s78, s19, 24
-; GFX11-NEXT: s_lshr_b32 s88, s19, 16
-; GFX11-NEXT: s_lshr_b32 s79, s19, 8
-; GFX11-NEXT: s_lshr_b32 s90, s18, 16
-; GFX11-NEXT: s_lshr_b32 s89, s18, 8
-; GFX11-NEXT: s_lshr_b32 s92, s17, 24
-; GFX11-NEXT: s_lshr_b32 s93, s17, 16
-; GFX11-NEXT: s_lshr_b32 s91, s17, 8
-; GFX11-NEXT: s_lshr_b32 s95, s16, 16
-; GFX11-NEXT: s_lshr_b32 s94, s16, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
-; GFX11-NEXT: s_lshr_b32 s30, s3, 16
-; GFX11-NEXT: s_lshr_b32 s31, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s35, s2, 8
-; GFX11-NEXT: s_lshr_b32 s36, s1, 24
-; GFX11-NEXT: s_lshr_b32 s37, s1, 16
-; GFX11-NEXT: s_lshr_b32 s38, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s48, s0, 8
-; GFX11-NEXT: .LBB25_3: ; %end
-; GFX11-NEXT: v_mov_b32_e32 v12, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v5, s34, s28, v12
-; GFX11-NEXT: v_readlane_b32 s34, v23, 0
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_perm_b32 v2, s39, s40, v12
-; GFX11-NEXT: v_perm_b32 v4, s37, s36, v12
-; GFX11-NEXT: v_perm_b32 v1, s0, s48, v12
-; GFX11-NEXT: v_perm_b32 v3, s1, s38, v12
-; GFX11-NEXT: v_perm_b32 v6, s2, s35, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_perm_b32 v8, s30, vcc_hi, v12
-; GFX11-NEXT: v_perm_b32 v9, s95, s14, v12
-; GFX11-NEXT: v_perm_b32 v10, s17, s91, v12
-; GFX11-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v2, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v6, v5
-; GFX11-NEXT: v_perm_b32 v5, s93, s92, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v8
-; GFX11-NEXT: v_perm_b32 v6, s16, s94, v12
-; GFX11-NEXT: v_perm_b32 v8, s90, s12, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-NEXT: v_perm_b32 v7, s3, s31, v12
-; GFX11-NEXT: v_perm_b32 v13, s18, s89, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-NEXT: v_or_b32_e32 v5, v6, v9
-; GFX11-NEXT: v_perm_b32 v9, s88, s78, v12
-; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
-; GFX11-NEXT: v_perm_b32 v10, s77, s10, v12
-; GFX11-NEXT: v_perm_b32 v11, s75, s74, v12
-; GFX11-NEXT: v_perm_b32 v14, s72, s8, v12
-; GFX11-NEXT: v_or_b32_e32 v4, v7, v4
-; GFX11-NEXT: v_or_b32_e32 v7, v13, v8
-; GFX11-NEXT: v_perm_b32 v8, s19, s79, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT: v_perm_b32 v13, s20, s76, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-NEXT: v_perm_b32 v15, s21, s73, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-NEXT: v_perm_b32 v16, s22, s63, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX11-NEXT: v_or_b32_e32 v9, v13, v10
-; GFX11-NEXT: v_perm_b32 v13, s62, s61, v12
-; GFX11-NEXT: v_or_b32_e32 v10, v15, v11
-; GFX11-NEXT: v_or_b32_e32 v11, v16, v14
-; GFX11-NEXT: v_perm_b32 v14, s59, s6, v12
-; GFX11-NEXT: v_perm_b32 v17, s57, s56, v12
-; GFX11-NEXT: v_perm_b32 v18, s46, s4, v12
-; GFX11-NEXT: v_perm_b32 v20, s44, s42, v12
-; GFX11-NEXT: v_perm_b32 v15, s23, s60, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-NEXT: v_perm_b32 v16, s24, s58, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-NEXT: v_perm_b32 v19, s25, s47, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-NEXT: v_perm_b32 v21, s26, s45, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-NEXT: v_perm_b32 v22, s27, s43, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-NEXT: v_or_b32_e32 v12, v15, v13
-; GFX11-NEXT: v_or_b32_e32 v13, v16, v14
-; GFX11-NEXT: v_or_b32_e32 v14, v19, v17
-; GFX11-NEXT: v_or_b32_e32 v15, v21, v18
-; GFX11-NEXT: v_or_b32_e32 v16, v22, v20
-; GFX11-NEXT: v_readlane_b32 s30, v23, 7
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off
-; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:32
-; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:48
-; GFX11-NEXT: v_readlane_b32 s31, v23, 8
-; GFX11-NEXT: v_readlane_b32 s48, v23, 6
-; GFX11-NEXT: v_readlane_b32 s39, v23, 5
-; GFX11-NEXT: v_readlane_b32 s38, v23, 4
-; GFX11-NEXT: v_readlane_b32 s37, v23, 3
-; GFX11-NEXT: v_readlane_b32 s36, v23, 2
-; GFX11-NEXT: v_readlane_b32 s35, v23, 1
-; GFX11-NEXT: s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT: scratch_load_b32 v23, off, s32 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_4:
-; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: s_branch .LBB25_3
+; GFX11-NEXT: .LBB25_2:
+; GFX11-NEXT: s_mov_b32 s48, -1
; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr40
; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr40
; GFX11-NEXT: ; implicit-def: $sgpr37
; GFX11-NEXT: ; implicit-def: $sgpr36
; GFX11-NEXT: ; implicit-def: $sgpr35
; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr28
; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr28
; GFX11-NEXT: ; implicit-def: $sgpr30
; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $vcc_lo
; GFX11-NEXT: ; implicit-def: $sgpr94
; GFX11-NEXT: ; implicit-def: $sgpr95
; GFX11-NEXT: ; implicit-def: $sgpr14
@@ -11341,7 +11455,163 @@ define inreg <64 x i8> @bitcast_v16i32_to_v64i8_scalar(<16 x i32> inreg %a, i32
; GFX11-NEXT: ; implicit-def: $sgpr43
; GFX11-NEXT: ; implicit-def: $sgpr44
; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: s_branch .LBB25_2
+; GFX11-NEXT: .LBB25_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s48, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: s_add_i32 s1, s1, 3
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s3, s3, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s17, s17, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s19, s19, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s21, s21, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s23, s23, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s25, s25, 3
+; GFX11-NEXT: s_add_i32 s27, s27, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[26:27], 24
+; GFX11-NEXT: s_lshr_b64 s[6:7], s[24:25], 24
+; GFX11-NEXT: s_lshr_b64 s[8:9], s[22:23], 24
+; GFX11-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
+; GFX11-NEXT: s_lshr_b64 s[12:13], s[18:19], 24
+; GFX11-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
+; GFX11-NEXT: s_lshr_b64 s[28:29], s[2:3], 24
+; GFX11-NEXT: s_lshr_b64 s[40:41], s[0:1], 24
+; GFX11-NEXT: s_lshr_b32 s42, s27, 24
+; GFX11-NEXT: s_lshr_b32 s44, s27, 16
+; GFX11-NEXT: s_lshr_b32 s43, s27, 8
+; GFX11-NEXT: s_lshr_b32 s46, s26, 16
+; GFX11-NEXT: s_lshr_b32 s45, s26, 8
+; GFX11-NEXT: s_lshr_b32 s56, s25, 24
+; GFX11-NEXT: s_lshr_b32 s57, s25, 16
+; GFX11-NEXT: s_lshr_b32 s47, s25, 8
+; GFX11-NEXT: s_lshr_b32 s59, s24, 16
+; GFX11-NEXT: s_lshr_b32 s58, s24, 8
+; GFX11-NEXT: s_lshr_b32 s61, s23, 24
+; GFX11-NEXT: s_lshr_b32 s62, s23, 16
+; GFX11-NEXT: s_lshr_b32 s60, s23, 8
+; GFX11-NEXT: s_lshr_b32 s72, s22, 16
+; GFX11-NEXT: s_lshr_b32 s63, s22, 8
+; GFX11-NEXT: s_lshr_b32 s74, s21, 24
+; GFX11-NEXT: s_lshr_b32 s75, s21, 16
+; GFX11-NEXT: s_lshr_b32 s73, s21, 8
+; GFX11-NEXT: s_lshr_b32 s77, s20, 16
+; GFX11-NEXT: s_lshr_b32 s76, s20, 8
+; GFX11-NEXT: s_lshr_b32 s78, s19, 24
+; GFX11-NEXT: s_lshr_b32 s88, s19, 16
+; GFX11-NEXT: s_lshr_b32 s79, s19, 8
+; GFX11-NEXT: s_lshr_b32 s90, s18, 16
+; GFX11-NEXT: s_lshr_b32 s89, s18, 8
+; GFX11-NEXT: s_lshr_b32 s92, s17, 24
+; GFX11-NEXT: s_lshr_b32 s93, s17, 16
+; GFX11-NEXT: s_lshr_b32 s91, s17, 8
+; GFX11-NEXT: s_lshr_b32 s95, s16, 16
+; GFX11-NEXT: s_lshr_b32 s94, s16, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s3, 24
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 16
+; GFX11-NEXT: s_lshr_b32 s30, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s34, s2, 8
+; GFX11-NEXT: s_lshr_b32 s35, s1, 24
+; GFX11-NEXT: s_lshr_b32 s36, s1, 16
+; GFX11-NEXT: s_lshr_b32 s37, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s39, s0, 8
+; GFX11-NEXT: .LBB25_5: ; %end
+; GFX11-NEXT: v_mov_b32_e32 v12, 0xc0c0004
+; GFX11-NEXT: v_readlane_b32 s48, v23, 6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v5, s31, s28, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_perm_b32 v2, s38, s40, v12
+; GFX11-NEXT: v_perm_b32 v4, s36, s35, v12
+; GFX11-NEXT: v_perm_b32 v1, s0, s39, v12
+; GFX11-NEXT: v_perm_b32 v3, s1, s37, v12
+; GFX11-NEXT: v_perm_b32 v6, s2, s34, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_perm_b32 v8, vcc_hi, vcc_lo, v12
+; GFX11-NEXT: v_perm_b32 v9, s95, s14, v12
+; GFX11-NEXT: v_perm_b32 v10, s17, s91, v12
+; GFX11-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v2, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v6, v5
+; GFX11-NEXT: v_perm_b32 v5, s93, s92, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX11-NEXT: v_perm_b32 v6, s16, s94, v12
+; GFX11-NEXT: v_perm_b32 v8, s90, s12, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-NEXT: v_perm_b32 v7, s3, s30, v12
+; GFX11-NEXT: v_perm_b32 v13, s18, s89, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-NEXT: v_or_b32_e32 v5, v6, v9
+; GFX11-NEXT: v_perm_b32 v9, s88, s78, v12
+; GFX11-NEXT: v_or_b32_e32 v6, v10, v11
+; GFX11-NEXT: v_perm_b32 v10, s77, s10, v12
+; GFX11-NEXT: v_perm_b32 v11, s75, s74, v12
+; GFX11-NEXT: v_perm_b32 v14, s72, s8, v12
+; GFX11-NEXT: v_or_b32_e32 v4, v7, v4
+; GFX11-NEXT: v_or_b32_e32 v7, v13, v8
+; GFX11-NEXT: v_perm_b32 v8, s19, s79, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-NEXT: v_perm_b32 v13, s20, s76, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT: v_perm_b32 v15, s21, s73, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-NEXT: v_perm_b32 v16, s22, s63, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX11-NEXT: v_or_b32_e32 v9, v13, v10
+; GFX11-NEXT: v_perm_b32 v13, s62, s61, v12
+; GFX11-NEXT: v_or_b32_e32 v10, v15, v11
+; GFX11-NEXT: v_or_b32_e32 v11, v16, v14
+; GFX11-NEXT: v_perm_b32 v14, s59, s6, v12
+; GFX11-NEXT: v_perm_b32 v17, s57, s56, v12
+; GFX11-NEXT: v_perm_b32 v18, s46, s4, v12
+; GFX11-NEXT: v_perm_b32 v20, s44, s42, v12
+; GFX11-NEXT: v_perm_b32 v15, s23, s60, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT: v_perm_b32 v16, s24, s58, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-NEXT: v_perm_b32 v19, s25, s47, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-NEXT: v_perm_b32 v21, s26, s45, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-NEXT: v_perm_b32 v22, s27, s43, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-NEXT: v_or_b32_e32 v12, v15, v13
+; GFX11-NEXT: v_or_b32_e32 v13, v16, v14
+; GFX11-NEXT: v_or_b32_e32 v14, v19, v17
+; GFX11-NEXT: v_or_b32_e32 v15, v21, v18
+; GFX11-NEXT: v_or_b32_e32 v16, v22, v20
+; GFX11-NEXT: v_readlane_b32 s30, v23, 7
+; GFX11-NEXT: s_clause 0x3
+; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off
+; GFX11-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
+; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:32
+; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:48
+; GFX11-NEXT: v_readlane_b32 s31, v23, 8
+; GFX11-NEXT: v_readlane_b32 s39, v23, 5
+; GFX11-NEXT: v_readlane_b32 s38, v23, 4
+; GFX11-NEXT: v_readlane_b32 s37, v23, 3
+; GFX11-NEXT: v_readlane_b32 s36, v23, 2
+; GFX11-NEXT: v_readlane_b32 s35, v23, 1
+; GFX11-NEXT: v_readlane_b32 s34, v23, 0
+; GFX11-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-NEXT: scratch_load_b32 v23, off, s32 ; 4-byte Folded Reload
+; GFX11-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -13709,7 +13979,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
; SI-NEXT: v_or_b32_e32 v0, s4, v34
@@ -13858,8 +14128,17 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s91
; SI-NEXT: v_mov_b32_e32 v9, s92
; SI-NEXT: v_mov_b32_e32 v10, s93
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -14070,12 +14349,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s15
; SI-NEXT: v_mov_b32_e32 v9, s11
; SI-NEXT: v_mov_b32_e32 v10, s7
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v64i8_to_v16i32_scalar:
; VI: ; %bb.0:
@@ -14134,7 +14410,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -14226,8 +14502,17 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v19, v18, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -14381,12 +14666,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v64i8_to_v16i32_scalar:
; GFX9: ; %bb.0:
@@ -14445,7 +14727,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -14539,8 +14821,17 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX9-NEXT: v_perm_b32 v35, v19, v18, s4
; GFX9-NEXT: v_or_b32_e32 v15, v35, v15
-; GFX9-NEXT: s_cbranch_execnz .LBB27_3
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -14681,12 +14972,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v15
; GFX9-NEXT: v_add_u32_sdwa v16, v16, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB27_3: ; %end
+; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB27_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16i32_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -14743,7 +15031,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -14815,9 +15103,17 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-TRUE16-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB27_3
+; GFX11-TRUE16-NEXT: .LBB27_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -14965,12 +15261,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-TRUE16-NEXT: .LBB27_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB27_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB27_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB27_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v16i32_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -15027,7 +15320,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -15099,9 +15392,17 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-FAKE16-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB27_3
+; GFX11-FAKE16-NEXT: .LBB27_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
@@ -15249,12 +15550,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-FAKE16-NEXT: .LBB27_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB27_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB27_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB27_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -15412,10 +15710,18 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v15, s31, 1.0
; SI-NEXT: v_add_f32_e64 v14, s30, 1.0
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -15432,10 +15738,8 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15452,7 +15756,7 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -15473,10 +15777,18 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -15493,10 +15805,8 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15513,7 +15823,7 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -15534,10 +15844,18 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -15554,10 +15872,8 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15574,7 +15890,7 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -15592,11 +15908,16 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -15614,8 +15935,6 @@ define inreg <8 x i64> @bitcast_v16f32_to_v8i64_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -15789,10 +16108,18 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s28, s28, 3
@@ -15809,7 +16136,7 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15827,8 +16154,6 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v8i64_to_v16f32_scalar:
; VI: ; %bb.0:
@@ -15837,10 +16162,18 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -15857,7 +16190,7 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15875,8 +16208,6 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v8i64_to_v16f32_scalar:
; GFX9: ; %bb.0:
@@ -15885,10 +16216,18 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_3
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -15905,7 +16244,7 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB31_3: ; %end
+; GFX9-NEXT: .LBB31_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15923,19 +16262,22 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_4:
-; GFX9-NEXT: s_branch .LBB31_2
;
; GFX11-LABEL: bitcast_v8i64_to_v16f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_3
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -15952,7 +16294,7 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB31_3: ; %end
+; GFX11-NEXT: .LBB31_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15963,8 +16305,6 @@ define inreg <16 x float> @bitcast_v8i64_to_v16f32_scalar(<8 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_4:
-; GFX11-NEXT: s_branch .LBB31_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16122,10 +16462,18 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v15, s31, 1.0
; SI-NEXT: v_add_f32_e64 v14, s30, 1.0
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -16142,10 +16490,8 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -16162,7 +16508,7 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16183,10 +16529,18 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -16203,10 +16557,8 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16223,7 +16575,7 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16244,10 +16596,18 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -16264,10 +16624,8 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16284,7 +16642,7 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16302,11 +16660,16 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -16324,8 +16687,6 @@ define inreg <8 x double> @bitcast_v16f32_to_v8f64_scalar(<16 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -16469,10 +16830,18 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -16481,10 +16850,8 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -16501,7 +16868,7 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: .LBB35_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16522,10 +16889,18 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -16534,10 +16909,8 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16554,7 +16927,7 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB35_5: ; %end
+; VI-NEXT: .LBB35_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16575,10 +16948,18 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -16587,10 +16968,8 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16607,7 +16986,7 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -16625,11 +17004,16 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -16639,8 +17023,6 @@ define inreg <16 x float> @bitcast_v8f64_to_v16f32_scalar(<8 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -16901,7 +17283,7 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s63, s5, 16
; SI-NEXT: s_lshr_b32 s62, s29, 16
@@ -16919,8 +17301,32 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v15, s5, 1.0
; SI-NEXT: v_add_f32_e64 v14, s4, 1.0
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -16953,26 +17359,8 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB37_5
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: s_branch .LBB37_6
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -17005,7 +17393,7 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v21, s40
; SI-NEXT: v_mov_b32_e32 v22, s42
; SI-NEXT: v_mov_b32_e32 v23, s44
-; SI-NEXT: .LBB37_5: ; %end
+; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
@@ -17068,10 +17456,18 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -17088,10 +17484,8 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB37_5
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: s_branch .LBB37_6
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17108,7 +17502,7 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB37_5: ; %end
+; VI-NEXT: .LBB37_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -17129,10 +17523,18 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -17149,10 +17551,8 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB37_5
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: s_branch .LBB37_6
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17169,7 +17569,7 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB37_5: ; %end
+; GFX9-NEXT: .LBB37_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -17187,11 +17587,16 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -17209,8 +17614,6 @@ define inreg <32 x i16> @bitcast_v16f32_to_v32i16_scalar(<16 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -17610,7 +18013,7 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s11, s12, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -17660,8 +18063,17 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s9, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -17742,7 +18154,7 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -17772,9 +18184,6 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v32i16_to_v16f32_scalar:
; VI: ; %bb.0:
@@ -17783,10 +18192,18 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s6, 3
; VI-NEXT: s_and_b32 s4, s6, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -17867,7 +18284,7 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17885,8 +18302,6 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v32i16_to_v16f32_scalar:
; GFX9: ; %bb.0:
@@ -17900,10 +18315,18 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -17920,10 +18343,8 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17940,7 +18361,7 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -17958,11 +18379,16 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -17980,8 +18406,6 @@ define inreg <16 x float> @bitcast_v32i16_to_v16f32_scalar(<32 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -18242,7 +18666,7 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s63, s5, 16
; SI-NEXT: s_lshr_b32 s62, s29, 16
@@ -18260,8 +18684,32 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v15, s5, 1.0
; SI-NEXT: v_add_f32_e64 v14, s4, 1.0
; SI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -18294,26 +18742,8 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -18346,7 +18776,7 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v21, s40
; SI-NEXT: v_mov_b32_e32 v22, s42
; SI-NEXT: v_mov_b32_e32 v23, s44
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
@@ -18409,10 +18839,18 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -18429,10 +18867,8 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB41_5
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: s_branch .LBB41_6
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -18449,7 +18885,7 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB41_5: ; %end
+; VI-NEXT: .LBB41_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -18470,10 +18906,18 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -18490,10 +18934,8 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB41_5
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: s_branch .LBB41_6
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -18510,7 +18952,7 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB41_5: ; %end
+; GFX9-NEXT: .LBB41_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -18528,11 +18970,16 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -18550,8 +18997,6 @@ define inreg <32 x half> @bitcast_v16f32_to_v32f16_scalar(<16 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -19015,7 +19460,7 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s10, s8, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_3
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -19065,8 +19510,17 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_4
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s62
@@ -19195,11 +19649,8 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v14, v15, v14
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: s_branch .LBB43_5
-; SI-NEXT: .LBB43_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB43_2
-; SI-NEXT: .LBB43_4:
+; SI-NEXT: s_branch .LBB43_6
+; SI-NEXT: .LBB43_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -19216,7 +19667,7 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: .LBB43_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v18, 7
; SI-NEXT: v_readlane_b32 s50, v18, 6
; SI-NEXT: v_readlane_b32 s49, v18, 5
@@ -19243,10 +19694,18 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s31, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -19328,10 +19787,8 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v16, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB43_5
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: s_branch .LBB43_6
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -19348,7 +19805,7 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: .LBB43_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v17, 0
; VI-NEXT: v_readlane_b32 s31, v17, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -19369,10 +19826,18 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -19390,10 +19855,8 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -19410,7 +19873,7 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -19428,11 +19891,16 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -19450,8 +19918,6 @@ define inreg <16 x float> @bitcast_v32f16_to_v16f32_scalar(<32 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -19792,7 +20258,7 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s8, s6, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s6, 16
@@ -19826,58 +20292,10 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; SI-NEXT: s_lshl_b32 s77, s17, 16
; SI-NEXT: s_and_b32 s78, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s79, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
-; SI-NEXT: v_add_f32_e64 v20, s21, 1.0
-; SI-NEXT: v_add_f32_e64 v18, s22, 1.0
-; SI-NEXT: v_add_f32_e64 v16, s23, 1.0
-; SI-NEXT: v_add_f32_e64 v14, s24, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
-; SI-NEXT: v_add_f32_e64 v8, s27, 1.0
-; SI-NEXT: v_add_f32_e64 v6, s28, 1.0
-; SI-NEXT: v_add_f32_e64 v4, s29, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s7, 1.0
-; SI-NEXT: v_add_f32_e64 v0, s6, 1.0
-; SI-NEXT: v_add_f32_e64 v22, s20, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s19, 1.0
-; SI-NEXT: v_add_f32_e64 v26, s18, 1.0
-; SI-NEXT: v_add_f32_e64 v28, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v30, s16, 1.0
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: ; implicit-def: $sgpr79
; SI-NEXT: ; implicit-def: $sgpr78
; SI-NEXT: ; implicit-def: $sgpr77
@@ -19910,8 +20328,62 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr9
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f32_e64 v20, s21, 1.0
+; SI-NEXT: v_add_f32_e64 v18, s22, 1.0
+; SI-NEXT: v_add_f32_e64 v16, s23, 1.0
+; SI-NEXT: v_add_f32_e64 v14, s24, 1.0
+; SI-NEXT: v_add_f32_e64 v12, s25, 1.0
+; SI-NEXT: v_add_f32_e64 v10, s26, 1.0
+; SI-NEXT: v_add_f32_e64 v8, s27, 1.0
+; SI-NEXT: v_add_f32_e64 v6, s28, 1.0
+; SI-NEXT: v_add_f32_e64 v4, s29, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s7, 1.0
+; SI-NEXT: v_add_f32_e64 v0, s6, 1.0
+; SI-NEXT: v_add_f32_e64 v22, s20, 1.0
+; SI-NEXT: v_add_f32_e64 v24, s19, 1.0
+; SI-NEXT: v_add_f32_e64 v26, s18, 1.0
+; SI-NEXT: v_add_f32_e64 v28, s17, 1.0
+; SI-NEXT: v_add_f32_e64 v30, s16, 1.0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v10
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v16
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v18
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v20
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v22
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v24
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v26
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v28
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
; SI-NEXT: v_mov_b32_e32 v30, s79
; SI-NEXT: v_mov_b32_e32 v31, s78
; SI-NEXT: v_mov_b32_e32 v28, s77
@@ -19944,7 +20416,7 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; SI-NEXT: v_mov_b32_e32 v3, s10
; SI-NEXT: v_mov_b32_e32 v0, s9
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
@@ -20023,10 +20495,18 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v15, s31, 1.0
; VI-NEXT: v_add_f32_e64 v14, s30, 1.0
; VI-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -20043,10 +20523,8 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
-; VI-NEXT: s_branch .LBB45_5
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: s_branch .LBB45_6
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -20063,7 +20541,7 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB45_5: ; %end
+; VI-NEXT: .LBB45_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -20084,10 +20562,18 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v15, s31, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s30, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s29, 1.0
@@ -20104,10 +20590,8 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
-; GFX9-NEXT: s_branch .LBB45_5
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: s_branch .LBB45_6
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20124,7 +20608,7 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB45_5: ; %end
+; GFX9-NEXT: .LBB45_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -20142,11 +20626,16 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v15, s27, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s26, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s25, 1.0
@@ -20164,8 +20653,6 @@ define inreg <32 x bfloat> @bitcast_v16f32_to_v32bf16_scalar(<16 x float> inreg
; GFX11-NEXT: v_add_f32_e64 v1, s13, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s12, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -21688,7 +22175,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
-; SI-NEXT: s_cbranch_scc0 .LBB47_4
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
@@ -21724,8 +22211,17 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v20, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
; SI-NEXT: v_mov_b32_e32 v16, v20
-; SI-NEXT: s_cbranch_execnz .LBB47_3
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -21822,7 +22318,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB47_3: ; %end
+; SI-NEXT: .LBB47_5: ; %end
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -21841,9 +22337,6 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB47_2
;
; VI-LABEL: bitcast_v32bf16_to_v16f32_scalar:
; VI: ; %bb.0:
@@ -21857,10 +22350,18 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s30, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v16
@@ -22158,10 +22659,8 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; VI-NEXT: v_lshrrev_b64 v[16:17], 16, v[17:18]
; VI-NEXT: v_mov_b32_e32 v13, v15
; VI-NEXT: v_mov_b32_e32 v15, v16
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -22178,7 +22677,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB47_5: ; %end
+; VI-NEXT: .LBB47_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -22199,10 +22698,18 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s31, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -22509,10 +23016,8 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -22529,7 +23034,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -22547,11 +23052,16 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-TRUE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s27, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -22861,8 +23371,6 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB47_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB47_2
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -22883,11 +23391,16 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-FAKE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s27, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s27, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -23216,8 +23729,6 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v16, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v20, 16, v17
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB47_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB47_2
; GFX11-FAKE16-NEXT: .LBB47_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -24735,14 +25246,14 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_writelane_b32 v40, s30, 28
; SI-NEXT: v_writelane_b32 v40, s31, 29
; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: v_readfirstlane_b32 s37, v2
+; SI-NEXT: v_readfirstlane_b32 s35, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_readfirstlane_b32 s36, v1
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: v_readfirstlane_b32 s34, v1
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s82, s37, 24
-; SI-NEXT: s_lshr_b32 s84, s37, 16
-; SI-NEXT: s_lshr_b32 s85, s37, 8
+; SI-NEXT: s_lshr_b32 s82, s35, 24
+; SI-NEXT: s_lshr_b32 s84, s35, 16
+; SI-NEXT: s_lshr_b32 s85, s35, 8
; SI-NEXT: s_lshr_b32 s71, s29, 24
; SI-NEXT: s_lshr_b32 s81, s29, 16
; SI-NEXT: s_lshr_b32 s83, s29, 8
@@ -24764,9 +25275,9 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: s_lshr_b32 s38, s17, 24
; SI-NEXT: s_lshr_b32 s39, s17, 16
; SI-NEXT: s_lshr_b32 s49, s17, 8
-; SI-NEXT: s_lshr_b64 s[4:5], s[36:37], 24
-; SI-NEXT: s_lshr_b64 s[6:7], s[36:37], 16
-; SI-NEXT: s_lshr_b64 s[8:9], s[36:37], 8
+; SI-NEXT: s_lshr_b64 s[4:5], s[34:35], 24
+; SI-NEXT: s_lshr_b64 s[6:7], s[34:35], 16
+; SI-NEXT: s_lshr_b64 s[8:9], s[34:35], 8
; SI-NEXT: s_lshr_b64 s[10:11], s[28:29], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[28:29], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[28:29], 8
@@ -24786,10 +25297,66 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 8
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v20, s17, 1.0
; SI-NEXT: v_add_f32_e64 v22, s16, 1.0
; SI-NEXT: v_add_f32_e64 v16, s19, 1.0
@@ -24804,8 +25371,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v6, s26, 1.0
; SI-NEXT: v_add_f32_e64 v3, s29, 1.0
; SI-NEXT: v_add_f32_e64 v4, s28, 1.0
-; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
-; SI-NEXT: v_add_f32_e64 v2, s36, 1.0
+; SI-NEXT: v_add_f32_e64 v1, s35, 1.0
+; SI-NEXT: v_add_f32_e64 v2, s34, 1.0
; SI-NEXT: v_readfirstlane_b32 s16, v22
; SI-NEXT: v_readfirstlane_b32 s17, v20
; SI-NEXT: v_readfirstlane_b32 s18, v18
@@ -24844,8 +25411,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 8
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 8
; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v1
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v1
; SI-NEXT: v_lshrrev_b32_e32 v14, 8, v1
@@ -24870,58 +25437,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v38, 24, v20
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v20
; SI-NEXT: v_lshrrev_b32_e32 v48, 8, v20
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v22, s16
; SI-NEXT: v_mov_b32_e32 v20, s17
; SI-NEXT: v_mov_b32_e32 v18, s18
@@ -24936,8 +25453,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v5, s27
; SI-NEXT: v_mov_b32_e32 v4, s28
; SI-NEXT: v_mov_b32_e32 v3, s29
-; SI-NEXT: v_mov_b32_e32 v2, s36
-; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v2, s34
+; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: v_mov_b32_e32 v48, s49
; SI-NEXT: v_mov_b32_e32 v39, s39
; SI-NEXT: v_mov_b32_e32 v38, s38
@@ -24962,11 +25479,11 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v14, s85
; SI-NEXT: v_mov_b32_e32 v13, s84
; SI-NEXT: v_mov_b32_e32 v12, s82
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_and_b32_e32 v22, 0xff, v22
-; SI-NEXT: s_lshl_b32 s5, s34, 8
+; SI-NEXT: s_lshl_b32 s5, s30, 8
; SI-NEXT: v_or_b32_e32 v22, s5, v22
-; SI-NEXT: s_and_b32 s5, s30, 0xff
+; SI-NEXT: s_and_b32 s5, vcc_lo, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s94, 24
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
@@ -25226,15 +25743,13 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_writelane_b32 v63, s55, 13
; VI-NEXT: v_writelane_b32 v63, s64, 14
; VI-NEXT: v_writelane_b32 v63, s65, 15
-; VI-NEXT: v_writelane_b32 v63, s66, 16
-; VI-NEXT: v_writelane_b32 v63, s67, 17
-; VI-NEXT: v_writelane_b32 v63, s30, 18
-; VI-NEXT: v_writelane_b32 v63, s31, 19
+; VI-NEXT: v_writelane_b32 v63, s30, 16
+; VI-NEXT: v_writelane_b32 v63, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s56, s5, 24
; VI-NEXT: s_lshr_b32 s57, s5, 16
@@ -25256,26 +25771,26 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: s_lshr_b32 s90, s25, 8
; VI-NEXT: s_lshr_b32 s89, s24, 16
; VI-NEXT: s_lshr_b32 s91, s24, 8
-; VI-NEXT: s_lshr_b32 s30, s23, 24
-; VI-NEXT: s_lshr_b32 s31, s23, 16
-; VI-NEXT: s_lshr_b32 s35, s23, 8
-; VI-NEXT: s_lshr_b32 s34, s22, 16
-; VI-NEXT: s_lshr_b32 s36, s22, 8
-; VI-NEXT: s_lshr_b32 s37, s21, 24
-; VI-NEXT: s_lshr_b32 s38, s21, 16
-; VI-NEXT: s_lshr_b32 s48, s21, 8
-; VI-NEXT: s_lshr_b32 s39, s20, 16
-; VI-NEXT: s_lshr_b32 s49, s20, 8
-; VI-NEXT: s_lshr_b32 s50, s19, 24
-; VI-NEXT: s_lshr_b32 s51, s19, 16
-; VI-NEXT: s_lshr_b32 s53, s19, 8
-; VI-NEXT: s_lshr_b32 s52, s18, 16
-; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s66, s17, 8
-; VI-NEXT: s_lshr_b32 s65, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s23, 16
+; VI-NEXT: s_lshr_b32 s31, s23, 8
+; VI-NEXT: s_lshr_b32 s30, s22, 16
+; VI-NEXT: s_lshr_b32 s34, s22, 8
+; VI-NEXT: s_lshr_b32 s35, s21, 24
+; VI-NEXT: s_lshr_b32 s36, s21, 16
+; VI-NEXT: s_lshr_b32 s38, s21, 8
+; VI-NEXT: s_lshr_b32 s37, s20, 16
+; VI-NEXT: s_lshr_b32 s39, s20, 8
+; VI-NEXT: s_lshr_b32 s48, s19, 24
+; VI-NEXT: s_lshr_b32 s49, s19, 16
+; VI-NEXT: s_lshr_b32 s51, s19, 8
+; VI-NEXT: s_lshr_b32 s50, s18, 16
+; VI-NEXT: s_lshr_b32 s52, s18, 8
+; VI-NEXT: s_lshr_b32 s53, s17, 24
+; VI-NEXT: s_lshr_b32 s54, s17, 16
+; VI-NEXT: s_lshr_b32 s64, s17, 8
+; VI-NEXT: s_lshr_b32 s55, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
; VI-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -25284,8 +25799,64 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v6, s27, 1.0
; VI-NEXT: v_add_f32_e64 v5, s26, 1.0
; VI-NEXT: v_add_f32_e64 v2, s5, 1.0
@@ -25354,58 +25925,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v62, 8, v16
; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v15
; VI-NEXT: v_lshrrev_b32_e32 v17, 8, v15
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v19, s44
; VI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -25426,26 +25947,26 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v4, s29
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: v_mov_b32_e32 v17, s67
-; VI-NEXT: v_mov_b32_e32 v18, s65
-; VI-NEXT: v_mov_b32_e32 v62, s66
-; VI-NEXT: v_mov_b32_e32 v60, s64
-; VI-NEXT: v_mov_b32_e32 v61, s55
-; VI-NEXT: v_mov_b32_e32 v58, s54
-; VI-NEXT: v_mov_b32_e32 v59, s52
-; VI-NEXT: v_mov_b32_e32 v57, s53
-; VI-NEXT: v_mov_b32_e32 v47, s51
-; VI-NEXT: v_mov_b32_e32 v56, s50
-; VI-NEXT: v_mov_b32_e32 v46, s49
-; VI-NEXT: v_mov_b32_e32 v45, s39
-; VI-NEXT: v_mov_b32_e32 v44, s48
-; VI-NEXT: v_mov_b32_e32 v42, s38
-; VI-NEXT: v_mov_b32_e32 v43, s37
-; VI-NEXT: v_mov_b32_e32 v41, s36
-; VI-NEXT: v_mov_b32_e32 v40, s34
-; VI-NEXT: v_mov_b32_e32 v55, s35
-; VI-NEXT: v_mov_b32_e32 v53, s31
-; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: v_mov_b32_e32 v17, s65
+; VI-NEXT: v_mov_b32_e32 v18, s55
+; VI-NEXT: v_mov_b32_e32 v62, s64
+; VI-NEXT: v_mov_b32_e32 v60, s54
+; VI-NEXT: v_mov_b32_e32 v61, s53
+; VI-NEXT: v_mov_b32_e32 v58, s52
+; VI-NEXT: v_mov_b32_e32 v59, s50
+; VI-NEXT: v_mov_b32_e32 v57, s51
+; VI-NEXT: v_mov_b32_e32 v47, s49
+; VI-NEXT: v_mov_b32_e32 v56, s48
+; VI-NEXT: v_mov_b32_e32 v46, s39
+; VI-NEXT: v_mov_b32_e32 v45, s37
+; VI-NEXT: v_mov_b32_e32 v44, s38
+; VI-NEXT: v_mov_b32_e32 v42, s36
+; VI-NEXT: v_mov_b32_e32 v43, s35
+; VI-NEXT: v_mov_b32_e32 v41, s34
+; VI-NEXT: v_mov_b32_e32 v40, s30
+; VI-NEXT: v_mov_b32_e32 v55, s31
+; VI-NEXT: v_mov_b32_e32 v53, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v54, vcc_lo
; VI-NEXT: v_mov_b32_e32 v52, s91
; VI-NEXT: v_mov_b32_e32 v51, s89
; VI-NEXT: v_mov_b32_e32 v50, s90
@@ -25474,7 +25995,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v19, s40
; VI-NEXT: v_mov_b32_e32 v20, s14
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v18, v18, v24, s4
; VI-NEXT: v_perm_b32 v15, v15, v17, s4
@@ -25551,10 +26072,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v3, v3, v34, s4
; VI-NEXT: v_perm_b32 v1, v1, v29, s4
-; VI-NEXT: v_readlane_b32 s30, v63, 18
-; VI-NEXT: v_readlane_b32 s31, v63, 19
-; VI-NEXT: v_readlane_b32 s67, v63, 17
-; VI-NEXT: v_readlane_b32 s66, v63, 16
+; VI-NEXT: v_readlane_b32 s30, v63, 16
+; VI-NEXT: v_readlane_b32 s31, v63, 17
; VI-NEXT: v_readlane_b32 s65, v63, 15
; VI-NEXT: v_readlane_b32 s64, v63, 14
; VI-NEXT: v_readlane_b32 s55, v63, 13
@@ -25651,15 +26170,13 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_writelane_b32 v63, s51, 9
; GFX9-NEXT: v_writelane_b32 v63, s52, 10
; GFX9-NEXT: v_writelane_b32 v63, s53, 11
-; GFX9-NEXT: v_writelane_b32 v63, s54, 12
-; GFX9-NEXT: v_writelane_b32 v63, s55, 13
-; GFX9-NEXT: v_writelane_b32 v63, s30, 14
-; GFX9-NEXT: v_writelane_b32 v63, s31, 15
+; GFX9-NEXT: v_writelane_b32 v63, s30, 12
+; GFX9-NEXT: v_writelane_b32 v63, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s56, s5, 24
; GFX9-NEXT: s_lshr_b32 s57, s5, 16
@@ -25685,22 +26202,22 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s95, s23, 8
; GFX9-NEXT: s_lshr_b32 s94, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s36, s21, 8
-; GFX9-NEXT: s_lshr_b32 s35, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s49, s19, 8
-; GFX9-NEXT: s_lshr_b32 s48, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s54, s17, 8
-; GFX9-NEXT: s_lshr_b32 s53, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s34, s21, 8
+; GFX9-NEXT: s_lshr_b32 s31, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s39, s19, 8
+; GFX9-NEXT: s_lshr_b32 s38, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s52, s17, 8
+; GFX9-NEXT: s_lshr_b32 s51, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
; GFX9-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -25709,8 +26226,64 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v6, s27, 1.0
; GFX9-NEXT: v_add_f32_e64 v5, s26, 1.0
; GFX9-NEXT: v_add_f32_e64 v2, s5, 1.0
@@ -25781,58 +26354,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v18
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v20, s44
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -25854,22 +26377,22 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v4, s29
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NEXT: v_mov_b32_e32 v15, s55
-; GFX9-NEXT: v_mov_b32_e32 v16, s53
-; GFX9-NEXT: v_mov_b32_e32 v62, s54
-; GFX9-NEXT: v_mov_b32_e32 v59, s52
-; GFX9-NEXT: v_mov_b32_e32 v60, s51
-; GFX9-NEXT: v_mov_b32_e32 v58, s50
-; GFX9-NEXT: v_mov_b32_e32 v61, s48
-; GFX9-NEXT: v_mov_b32_e32 v57, s49
-; GFX9-NEXT: v_mov_b32_e32 v47, s39
-; GFX9-NEXT: v_mov_b32_e32 v56, s38
-; GFX9-NEXT: v_mov_b32_e32 v46, s37
-; GFX9-NEXT: v_mov_b32_e32 v45, s35
-; GFX9-NEXT: v_mov_b32_e32 v44, s36
-; GFX9-NEXT: v_mov_b32_e32 v42, s34
-; GFX9-NEXT: v_mov_b32_e32 v43, s31
-; GFX9-NEXT: v_mov_b32_e32 v41, s30
+; GFX9-NEXT: v_mov_b32_e32 v15, s53
+; GFX9-NEXT: v_mov_b32_e32 v16, s51
+; GFX9-NEXT: v_mov_b32_e32 v62, s52
+; GFX9-NEXT: v_mov_b32_e32 v59, s50
+; GFX9-NEXT: v_mov_b32_e32 v60, s49
+; GFX9-NEXT: v_mov_b32_e32 v58, s48
+; GFX9-NEXT: v_mov_b32_e32 v61, s38
+; GFX9-NEXT: v_mov_b32_e32 v57, s39
+; GFX9-NEXT: v_mov_b32_e32 v47, s37
+; GFX9-NEXT: v_mov_b32_e32 v56, s36
+; GFX9-NEXT: v_mov_b32_e32 v46, s35
+; GFX9-NEXT: v_mov_b32_e32 v45, s31
+; GFX9-NEXT: v_mov_b32_e32 v44, s34
+; GFX9-NEXT: v_mov_b32_e32 v42, s30
+; GFX9-NEXT: v_mov_b32_e32 v43, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v41, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v40, s94
; GFX9-NEXT: v_mov_b32_e32 v55, s95
; GFX9-NEXT: v_mov_b32_e32 v53, s93
@@ -25903,7 +26426,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v20, s40
; GFX9-NEXT: v_mov_b32_e32 v21, s14
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v16, v25, s4
; GFX9-NEXT: v_perm_b32 v15, v18, v15, s4
@@ -25970,10 +26493,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
-; GFX9-NEXT: v_readlane_b32 s30, v63, 14
-; GFX9-NEXT: v_readlane_b32 s31, v63, 15
-; GFX9-NEXT: v_readlane_b32 s55, v63, 13
-; GFX9-NEXT: v_readlane_b32 s54, v63, 12
+; GFX9-NEXT: v_readlane_b32 s30, v63, 12
+; GFX9-NEXT: v_readlane_b32 s31, v63, 13
; GFX9-NEXT: v_readlane_b32 s53, v63, 11
; GFX9-NEXT: v_readlane_b32 s52, v63, 10
; GFX9-NEXT: v_readlane_b32 s51, v63, 9
@@ -26043,12 +26564,11 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s43, s27, 24
; GFX11-NEXT: s_lshr_b32 s44, s27, 16
@@ -26079,17 +26599,17 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s93, s17, 16
; GFX11-NEXT: s_lshr_b32 s95, s17, 8
; GFX11-NEXT: s_lshr_b32 s94, s16, 16
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s16, 8
-; GFX11-NEXT: s_lshr_b32 s30, s3, 24
-; GFX11-NEXT: s_lshr_b32 s31, s3, 16
-; GFX11-NEXT: s_lshr_b32 s35, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s36, s2, 8
-; GFX11-NEXT: s_lshr_b32 s37, s1, 24
-; GFX11-NEXT: s_lshr_b32 s38, s1, 16
-; GFX11-NEXT: s_lshr_b32 s48, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s49, s0, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s16, 8
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
+; GFX11-NEXT: s_lshr_b32 s30, s3, 16
+; GFX11-NEXT: s_lshr_b32 s34, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s35, s2, 8
+; GFX11-NEXT: s_lshr_b32 s36, s1, 24
+; GFX11-NEXT: s_lshr_b32 s37, s1, 16
+; GFX11-NEXT: s_lshr_b32 s39, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s48, s0, 8
; GFX11-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[14:15], s[22:23], 24
@@ -26098,9 +26618,64 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB49_3
+; GFX11-NEXT: .LBB49_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr95
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: .LBB49_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v4, s25, 1.0
; GFX11-NEXT: v_add_f32_e64 v3, s24, 1.0
; GFX11-NEXT: v_add_f32_e64 v6, s23, 1.0
@@ -26165,58 +26740,8 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_lshrrev_b32_e32 v83, 8, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v96, 16, v17
; GFX11-NEXT: v_lshrrev_b32_e32 v85, 8, v17
-; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr31
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr95
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: s_branch .LBB49_2
-; GFX11-NEXT: .LBB49_4:
+; GFX11-NEXT: s_branch .LBB49_6
+; GFX11-NEXT: .LBB49_5:
; GFX11-NEXT: v_dual_mov_b32 v17, s0 :: v_dual_mov_b32 v18, s1
; GFX11-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s3
; GFX11-NEXT: v_dual_mov_b32 v11, s16 :: v_dual_mov_b32 v12, s17
@@ -26225,12 +26750,12 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v5, s22 :: v_dual_mov_b32 v6, s23
; GFX11-NEXT: v_dual_mov_b32 v3, s24 :: v_dual_mov_b32 v4, s25
; GFX11-NEXT: v_dual_mov_b32 v1, s26 :: v_dual_mov_b32 v2, s27
-; GFX11-NEXT: v_dual_mov_b32 v85, s49 :: v_dual_mov_b32 v96, s39
-; GFX11-NEXT: v_dual_mov_b32 v83, s48 :: v_dual_mov_b32 v86, s38
-; GFX11-NEXT: v_dual_mov_b32 v87, s37 :: v_dual_mov_b32 v82, s36
-; GFX11-NEXT: v_dual_mov_b32 v84, s34 :: v_dual_mov_b32 v71, s35
-; GFX11-NEXT: v_dual_mov_b32 v80, s31 :: v_dual_mov_b32 v81, s30
-; GFX11-NEXT: v_dual_mov_b32 v67, vcc_hi :: v_dual_mov_b32 v70, s94
+; GFX11-NEXT: v_dual_mov_b32 v85, s48 :: v_dual_mov_b32 v96, s38
+; GFX11-NEXT: v_dual_mov_b32 v83, s39 :: v_dual_mov_b32 v86, s37
+; GFX11-NEXT: v_dual_mov_b32 v87, s36 :: v_dual_mov_b32 v82, s35
+; GFX11-NEXT: v_dual_mov_b32 v84, s31 :: v_dual_mov_b32 v71, s34
+; GFX11-NEXT: v_dual_mov_b32 v80, s30 :: v_dual_mov_b32 v81, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v67, vcc_lo :: v_dual_mov_b32 v70, s94
; GFX11-NEXT: v_dual_mov_b32 v65, s95 :: v_dual_mov_b32 v68, s93
; GFX11-NEXT: v_dual_mov_b32 v69, s92 :: v_dual_mov_b32 v64, s91
; GFX11-NEXT: v_dual_mov_b32 v66, s89 :: v_dual_mov_b32 v53, s90
@@ -26249,7 +26774,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v20, s14 :: v_dual_mov_b32 v21, s12
; GFX11-NEXT: v_dual_mov_b32 v22, s10 :: v_dual_mov_b32 v23, s8
; GFX11-NEXT: v_dual_mov_b32 v24, s6 :: v_dual_mov_b32 v25, s4
-; GFX11-NEXT: .LBB49_5: ; %end
+; GFX11-NEXT: .LBB49_6: ; %end
; GFX11-NEXT: v_perm_b32 v86, v86, v87, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_perm_b32 v24, v84, v24, 0xc0c0004
@@ -26315,14 +26840,13 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v2, v4, v17
; GFX11-NEXT: v_or_b32_e32 v3, v19, v15
; GFX11-NEXT: v_or_b32_e32 v4, v16, v18
-; GFX11-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: scratch_store_b128 v0, v[80:83], off
; GFX11-NEXT: scratch_store_b128 v0, v[11:14], off offset:16
; GFX11-NEXT: scratch_store_b128 v0, v[7:10], off offset:32
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:48
-; GFX11-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-NEXT: v_readlane_b32 s38, v40, 4
@@ -28702,7 +29226,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
; SI-NEXT: v_or_b32_e32 v0, s4, v34
@@ -28851,8 +29375,17 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s91
; SI-NEXT: v_mov_b32_e32 v9, s92
; SI-NEXT: v_mov_b32_e32 v10, s93
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -29063,12 +29596,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, s15
; SI-NEXT: v_mov_b32_e32 v9, s11
; SI-NEXT: v_mov_b32_e32 v10, s7
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v64i8_to_v16f32_scalar:
; VI: ; %bb.0:
@@ -29127,7 +29657,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -29219,8 +29749,17 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v19, v18, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -29374,12 +29913,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v64i8_to_v16f32_scalar:
; GFX9: ; %bb.0:
@@ -29438,7 +29974,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -29532,8 +30068,17 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX9-NEXT: v_perm_b32 v35, v19, v18, s4
; GFX9-NEXT: v_or_b32_e32 v15, v35, v15
-; GFX9-NEXT: s_cbranch_execnz .LBB51_3
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -29674,12 +30219,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v15
; GFX9-NEXT: v_add_u32_sdwa v16, v16, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB51_3: ; %end
+; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB51_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v16f32_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -29736,7 +30278,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -29808,9 +30350,17 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-TRUE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB51_3
+; GFX11-TRUE16-NEXT: .LBB51_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -29958,12 +30508,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-TRUE16-NEXT: .LBB51_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB51_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB51_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v16f32_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -30020,7 +30567,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -30092,9 +30639,17 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-FAKE16-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB51_3
+; GFX11-FAKE16-NEXT: .LBB51_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
@@ -30242,12 +30797,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-FAKE16-NEXT: .LBB51_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB51_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB51_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -30411,10 +30963,18 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -30431,7 +30991,7 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s29, s29, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -30449,8 +31009,6 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s7
; SI-NEXT: v_mov_b32_e32 v15, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v8i64_to_v8f64_scalar:
; VI: ; %bb.0:
@@ -30459,10 +31017,18 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -30479,7 +31045,7 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s29, s29, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -30497,8 +31063,6 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v8i64_to_v8f64_scalar:
; GFX9: ; %bb.0:
@@ -30507,10 +31071,18 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -30527,7 +31099,7 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s29, s29, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -30545,19 +31117,22 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v8i64_to_v8f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -30574,7 +31149,7 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s25, s25, 0
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -30584,8 +31159,6 @@ define inreg <8 x double> @bitcast_v8i64_to_v8f64_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -30719,10 +31292,18 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s31, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s30, v0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -30731,10 +31312,8 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -30751,7 +31330,7 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v14, s30
; SI-NEXT: v_mov_b32_e32 v15, s31
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s30, v16, 0
; SI-NEXT: v_readlane_b32 s31, v16, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -30772,10 +31351,18 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -30784,10 +31371,8 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -30804,7 +31389,7 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB55_5: ; %end
+; VI-NEXT: .LBB55_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -30825,10 +31410,18 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
@@ -30837,10 +31430,8 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -30857,7 +31448,7 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -30875,11 +31466,16 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[16:17], 1.0
@@ -30889,8 +31485,6 @@ define inreg <8 x i64> @bitcast_v8f64_to_v8i64_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -31163,7 +31757,7 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s5, 16
; SI-NEXT: s_lshr_b32 s57, s29, 16
@@ -31181,8 +31775,32 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s28, s28, 3
@@ -31215,7 +31833,7 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s7, s16, 0xffff
; SI-NEXT: s_lshl_b32 s9, s44, 16
; SI-NEXT: s_or_b32 s7, s7, s9
@@ -31281,24 +31899,6 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v8i64_to_v32i16_scalar:
; VI: ; %bb.0:
@@ -31307,10 +31907,18 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -31327,7 +31935,7 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -31345,8 +31953,6 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v8i64_to_v32i16_scalar:
; GFX9: ; %bb.0:
@@ -31355,10 +31961,18 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_3
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -31375,7 +31989,7 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB57_3: ; %end
+; GFX9-NEXT: .LBB57_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -31393,19 +32007,22 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: s_branch .LBB57_2
;
; GFX11-LABEL: bitcast_v8i64_to_v32i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_3
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -31422,7 +32039,7 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB57_3: ; %end
+; GFX11-NEXT: .LBB57_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -31433,8 +32050,6 @@ define inreg <32 x i16> @bitcast_v8i64_to_v32i16_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_4:
-; GFX11-NEXT: s_branch .LBB57_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -31824,7 +32439,7 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s11, s12, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -31874,8 +32489,17 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s9, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -31956,7 +32580,7 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -31986,9 +32610,6 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v32i16_to_v8i64_scalar:
; VI: ; %bb.0:
@@ -31997,10 +32618,18 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s6, 3
; VI-NEXT: s_and_b32 s4, s6, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -32081,7 +32710,7 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -32099,8 +32728,6 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v32i16_to_v8i64_scalar:
; GFX9: ; %bb.0:
@@ -32114,10 +32741,18 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -32134,10 +32769,8 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -32154,7 +32787,7 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -32172,11 +32805,16 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -32194,8 +32832,6 @@ define inreg <8 x i64> @bitcast_v32i16_to_v8i64_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -32468,7 +33104,7 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB61_4
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s56, s5, 16
; SI-NEXT: s_lshr_b32 s57, s29, 16
@@ -32486,8 +33122,32 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB61_3
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s28, s28, 3
@@ -32520,7 +33180,7 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: .LBB61_3: ; %end
+; SI-NEXT: .LBB61_5: ; %end
; SI-NEXT: s_and_b32 s7, s16, 0xffff
; SI-NEXT: s_lshl_b32 s9, s44, 16
; SI-NEXT: s_or_b32 s7, s7, s9
@@ -32586,24 +33246,6 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB61_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: s_branch .LBB61_2
;
; VI-LABEL: bitcast_v8i64_to_v32f16_scalar:
; VI: ; %bb.0:
@@ -32612,10 +33254,18 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB61_4
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_3
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -32632,7 +33282,7 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB61_3: ; %end
+; VI-NEXT: .LBB61_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -32650,8 +33300,6 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_4:
-; VI-NEXT: s_branch .LBB61_2
;
; GFX9-LABEL: bitcast_v8i64_to_v32f16_scalar:
; GFX9: ; %bb.0:
@@ -32660,10 +33308,18 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_3
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -32680,7 +33336,7 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB61_3: ; %end
+; GFX9-NEXT: .LBB61_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -32698,19 +33354,22 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_4:
-; GFX9-NEXT: s_branch .LBB61_2
;
; GFX11-LABEL: bitcast_v8i64_to_v32f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_3
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -32727,7 +33386,7 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB61_3: ; %end
+; GFX11-NEXT: .LBB61_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -32738,8 +33397,6 @@ define inreg <32 x half> @bitcast_v8i64_to_v32f16_scalar(<8 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_4:
-; GFX11-NEXT: s_branch .LBB61_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -33193,7 +33850,7 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s10, s8, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB63_3
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -33243,8 +33900,17 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB63_4
-; SI-NEXT: .LBB63_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s62
@@ -33373,11 +34039,8 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_or_b32_e32 v14, v15, v14
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: s_branch .LBB63_5
-; SI-NEXT: .LBB63_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB63_2
-; SI-NEXT: .LBB63_4:
+; SI-NEXT: s_branch .LBB63_6
+; SI-NEXT: .LBB63_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -33394,7 +34057,7 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB63_5: ; %end
+; SI-NEXT: .LBB63_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v18, 7
; SI-NEXT: v_readlane_b32 s50, v18, 6
; SI-NEXT: v_readlane_b32 s49, v18, 5
@@ -33421,10 +34084,18 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s31, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -33506,10 +34177,8 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_add_f16_sdwa v16, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB63_5
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: s_branch .LBB63_6
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -33526,7 +34195,7 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB63_5: ; %end
+; VI-NEXT: .LBB63_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v17, 0
; VI-NEXT: v_readlane_b32 s31, v17, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -33547,10 +34216,18 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -33568,10 +34245,8 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB63_5
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: s_branch .LBB63_6
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -33588,7 +34263,7 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB63_5: ; %end
+; GFX9-NEXT: .LBB63_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -33606,11 +34281,16 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB63_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -33628,8 +34308,6 @@ define inreg <8 x i64> @bitcast_v32f16_to_v8i64_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB63_3:
-; GFX11-NEXT: s_branch .LBB63_2
; GFX11-NEXT: .LBB63_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -33982,7 +34660,7 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s78, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s79, v0
-; SI-NEXT: s_cbranch_scc0 .LBB65_4
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s78, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s78, 16
@@ -34016,8 +34694,48 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; SI-NEXT: s_lshl_b32 s74, s17, 16
; SI-NEXT: s_and_b32 s77, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s76, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB65_3
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s16, 3
; SI-NEXT: s_addc_u32 s5, s17, 0
; SI-NEXT: s_add_u32 s16, s18, 3
@@ -34066,7 +34784,7 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; SI-NEXT: s_lshl_b32 s74, s5, 16
; SI-NEXT: s_and_b32 s77, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s76, s4, 16
-; SI-NEXT: .LBB65_3: ; %end
+; SI-NEXT: .LBB65_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
@@ -34132,40 +34850,6 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB65_2
;
; VI-LABEL: bitcast_v8i64_to_v32bf16_scalar:
; VI: ; %bb.0:
@@ -34174,10 +34858,18 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB65_4
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_3
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s28, s28, 3
@@ -34194,7 +34886,7 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB65_3: ; %end
+; VI-NEXT: .LBB65_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -34212,8 +34904,6 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_4:
-; VI-NEXT: s_branch .LBB65_2
;
; GFX9-LABEL: bitcast_v8i64_to_v32bf16_scalar:
; GFX9: ; %bb.0:
@@ -34222,10 +34912,18 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_3
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s28, s28, 3
@@ -34242,7 +34940,7 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB65_3: ; %end
+; GFX9-NEXT: .LBB65_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -34260,19 +34958,22 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v14, s7
; GFX9-NEXT: v_mov_b32_e32 v15, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_4:
-; GFX9-NEXT: s_branch .LBB65_2
;
; GFX11-LABEL: bitcast_v8i64_to_v32bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_3
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s26, s26, 3
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s24, s24, 3
@@ -34289,7 +34990,7 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB65_3: ; %end
+; GFX11-NEXT: .LBB65_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -34300,8 +35001,6 @@ define inreg <32 x bfloat> @bitcast_v8i64_to_v32bf16_scalar(<8 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_4:
-; GFX11-NEXT: s_branch .LBB65_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -35814,7 +36513,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
-; SI-NEXT: s_cbranch_scc0 .LBB67_4
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
@@ -35850,8 +36549,17 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
; SI-NEXT: v_mov_b32_e32 v16, v20
-; SI-NEXT: s_cbranch_execnz .LBB67_3
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -35948,7 +36656,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB67_3: ; %end
+; SI-NEXT: .LBB67_5: ; %end
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -35967,9 +36675,6 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB67_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB67_2
;
; VI-LABEL: bitcast_v32bf16_to_v8i64_scalar:
; VI: ; %bb.0:
@@ -35983,10 +36688,18 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s30, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v16
@@ -36284,10 +36997,8 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_lshrrev_b64 v[16:17], 16, v[17:18]
; VI-NEXT: v_mov_b32_e32 v13, v15
; VI-NEXT: v_mov_b32_e32 v15, v16
-; VI-NEXT: s_branch .LBB67_5
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: s_branch .LBB67_6
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -36304,7 +37015,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB67_5: ; %end
+; VI-NEXT: .LBB67_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -36325,10 +37036,18 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s31, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -36635,10 +37354,8 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
-; GFX9-NEXT: s_branch .LBB67_5
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: s_branch .LBB67_6
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -36655,7 +37372,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB67_5: ; %end
+; GFX9-NEXT: .LBB67_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -36673,11 +37390,16 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-TRUE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s27, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -36987,8 +37709,6 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB67_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB67_2
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -37009,11 +37729,16 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-FAKE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s27, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s27, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -37342,8 +38067,6 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v16, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v20, 16, v17
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB67_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB67_2
; GFX11-FAKE16-NEXT: .LBB67_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -38890,7 +39613,7 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s5, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_cbranch_scc0 .LBB69_4
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s38, s5, 24
; SI-NEXT: s_lshr_b32 s39, s5, 16
@@ -38937,11 +39660,67 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB69_3
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[36:37], -1
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr83
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr71
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr65
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s28, s28, 3
@@ -39003,15 +39782,15 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 8
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 24
-; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[16:17], 8
-; SI-NEXT: .LBB69_3: ; %end
-; SI-NEXT: s_lshl_b32 s7, s36, 8
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 24
+; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[16:17], 8
+; SI-NEXT: .LBB69_5: ; %end
+; SI-NEXT: s_lshl_b32 s7, s34, 8
; SI-NEXT: s_and_b32 s9, s16, 0xff
; SI-NEXT: s_or_b32 s7, s9, s7
-; SI-NEXT: s_and_b32 s9, s34, 0xff
-; SI-NEXT: s_lshl_b32 s11, s30, 24
+; SI-NEXT: s_and_b32 s9, s30, 0xff
+; SI-NEXT: s_lshl_b32 s11, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_or_b32 s9, s11, s9
; SI-NEXT: s_and_b32 s7, s7, 0xffff
@@ -39246,56 +40025,6 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_4:
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr85
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB69_2
;
; VI-LABEL: bitcast_v8i64_to_v64i8_scalar:
; VI: ; %bb.0:
@@ -39319,15 +40048,13 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: v_writelane_b32 v4, s55, 13
; VI-NEXT: v_writelane_b32 v4, s64, 14
; VI-NEXT: v_writelane_b32 v4, s65, 15
-; VI-NEXT: v_writelane_b32 v4, s66, 16
-; VI-NEXT: v_writelane_b32 v4, s67, 17
-; VI-NEXT: v_writelane_b32 v4, s30, 18
-; VI-NEXT: v_writelane_b32 v4, s31, 19
+; VI-NEXT: v_writelane_b32 v4, s30, 16
+; VI-NEXT: v_writelane_b32 v4, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB69_4
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s56, s5, 24
; VI-NEXT: s_lshr_b32 s57, s5, 16
@@ -39364,11 +40091,11 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[26:27], 24
@@ -39377,8 +40104,64 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[44:45], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB69_3
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -39438,23 +40221,23 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
-; VI-NEXT: .LBB69_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
+; VI-NEXT: .LBB69_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s44
-; VI-NEXT: v_mov_b32_e32 v1, s67
-; VI-NEXT: v_perm_b32 v3, s66, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s65
+; VI-NEXT: v_perm_b32 v3, s64, v3, v2
; VI-NEXT: v_perm_b32 v1, s16, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
-; VI-NEXT: v_mov_b32_e32 v3, s55
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v1, s65
-; VI-NEXT: v_perm_b32 v3, s64, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s55
+; VI-NEXT: v_perm_b32 v3, vcc_hi, v3, v2
; VI-NEXT: v_perm_b32 v1, s17, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -39571,11 +40354,9 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: v_add_u32_e32 v0, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s30, v4, 18
+; VI-NEXT: v_readlane_b32 s30, v4, 16
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_readlane_b32 s31, v4, 19
-; VI-NEXT: v_readlane_b32 s67, v4, 17
-; VI-NEXT: v_readlane_b32 s66, v4, 16
+; VI-NEXT: v_readlane_b32 s31, v4, 17
; VI-NEXT: v_readlane_b32 s65, v4, 15
; VI-NEXT: v_readlane_b32 s64, v4, 14
; VI-NEXT: v_readlane_b32 s55, v4, 13
@@ -39597,56 +40378,6 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_4:
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB69_2
;
; GFX9-LABEL: bitcast_v8i64_to_v64i8_scalar:
; GFX9: ; %bb.0:
@@ -39666,15 +40397,13 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: v_writelane_b32 v4, s51, 9
; GFX9-NEXT: v_writelane_b32 v4, s52, 10
; GFX9-NEXT: v_writelane_b32 v4, s53, 11
-; GFX9-NEXT: v_writelane_b32 v4, s54, 12
-; GFX9-NEXT: v_writelane_b32 v4, s55, 13
-; GFX9-NEXT: v_writelane_b32 v4, s30, 14
-; GFX9-NEXT: v_writelane_b32 v4, s31, 15
+; GFX9-NEXT: v_writelane_b32 v4, s30, 12
+; GFX9-NEXT: v_writelane_b32 v4, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s56, s5, 24
; GFX9-NEXT: s_lshr_b32 s57, s5, 16
@@ -39700,22 +40429,22 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s94, s23, 8
; GFX9-NEXT: s_lshr_b32 s95, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s35, s21, 8
-; GFX9-NEXT: s_lshr_b32 s36, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s48, s19, 8
-; GFX9-NEXT: s_lshr_b32 s49, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s53, s17, 8
-; GFX9-NEXT: s_lshr_b32 s54, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s31, s21, 8
+; GFX9-NEXT: s_lshr_b32 s34, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s38, s19, 8
+; GFX9-NEXT: s_lshr_b32 s39, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s51, s17, 8
+; GFX9-NEXT: s_lshr_b32 s52, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[10:11], s[26:27], 24
@@ -39724,8 +40453,64 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[44:45], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB69_3
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -39774,68 +40559,68 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s94, s23, 8
; GFX9-NEXT: s_lshr_b32 s95, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s35, s21, 8
-; GFX9-NEXT: s_lshr_b32 s36, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s48, s19, 8
-; GFX9-NEXT: s_lshr_b32 s49, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s53, s17, 8
-; GFX9-NEXT: s_lshr_b32 s54, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
-; GFX9-NEXT: .LBB69_3: ; %end
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s31, s21, 8
+; GFX9-NEXT: s_lshr_b32 s34, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s38, s19, 8
+; GFX9-NEXT: s_lshr_b32 s39, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s51, s17, 8
+; GFX9-NEXT: s_lshr_b32 s52, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
+; GFX9-NEXT: .LBB69_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v3, s44
-; GFX9-NEXT: v_mov_b32_e32 v1, s55
-; GFX9-NEXT: v_perm_b32 v3, s54, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s53
+; GFX9-NEXT: v_perm_b32 v3, s52, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s16, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s51
+; GFX9-NEXT: v_mov_b32_e32 v3, s49
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; GFX9-NEXT: v_mov_b32_e32 v1, s53
-; GFX9-NEXT: v_perm_b32 v3, s52, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s51
+; GFX9-NEXT: v_perm_b32 v3, s50, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s17, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s42
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT: v_mov_b32_e32 v1, s50
-; GFX9-NEXT: v_perm_b32 v3, s49, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s48
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s18, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s38
+; GFX9-NEXT: v_mov_b32_e32 v3, s36
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
-; GFX9-NEXT: v_mov_b32_e32 v1, s48
-; GFX9-NEXT: v_perm_b32 v3, s39, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s38
+; GFX9-NEXT: v_perm_b32 v3, s37, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s19, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s40
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:12
-; GFX9-NEXT: v_mov_b32_e32 v1, s37
-; GFX9-NEXT: v_perm_b32 v3, s36, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s35
+; GFX9-NEXT: v_perm_b32 v3, s34, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s20, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s31
+; GFX9-NEXT: v_mov_b32_e32 v3, vcc_hi
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
-; GFX9-NEXT: v_mov_b32_e32 v1, s35
-; GFX9-NEXT: v_perm_b32 v3, s34, v3, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s31
+; GFX9-NEXT: v_perm_b32 v3, s30, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s21, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_mov_b32_e32 v1, s30
+; GFX9-NEXT: v_mov_b32_e32 v1, vcc_lo
; GFX9-NEXT: v_perm_b32 v3, s95, v3, v2
; GFX9-NEXT: v_perm_b32 v1, s22, v1, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
@@ -39903,11 +40688,9 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v2, s57, v3, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_readlane_b32 s30, v4, 14
+; GFX9-NEXT: v_readlane_b32 s30, v4, 12
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
-; GFX9-NEXT: v_readlane_b32 s31, v4, 15
-; GFX9-NEXT: v_readlane_b32 s55, v4, 13
-; GFX9-NEXT: v_readlane_b32 s54, v4, 12
+; GFX9-NEXT: v_readlane_b32 s31, v4, 13
; GFX9-NEXT: v_readlane_b32 s53, v4, 11
; GFX9-NEXT: v_readlane_b32 s52, v4, 10
; GFX9-NEXT: v_readlane_b32 s51, v4, 9
@@ -39925,56 +40708,6 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_4:
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: s_branch .LBB69_2
;
; GFX11-LABEL: bitcast_v8i64_to_v64i8_scalar:
; GFX11: ; %bb.0:
@@ -39992,8 +40725,8 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: v_writelane_b32 v23, s30, 7
; GFX11-NEXT: v_writelane_b32 v23, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 vcc_lo, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX11-NEXT: s_mov_b32 s48, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s42, s27, 24
; GFX11-NEXT: s_lshr_b32 s44, s27, 16
@@ -40025,16 +40758,16 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s91, s17, 8
; GFX11-NEXT: s_lshr_b32 s95, s16, 16
; GFX11-NEXT: s_lshr_b32 s94, s16, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
-; GFX11-NEXT: s_lshr_b32 s30, s3, 16
-; GFX11-NEXT: s_lshr_b32 s31, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s35, s2, 8
-; GFX11-NEXT: s_lshr_b32 s36, s1, 24
-; GFX11-NEXT: s_lshr_b32 s37, s1, 16
-; GFX11-NEXT: s_lshr_b32 s38, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s48, s0, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s3, 24
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 16
+; GFX11-NEXT: s_lshr_b32 s30, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s34, s2, 8
+; GFX11-NEXT: s_lshr_b32 s35, s1, 24
+; GFX11-NEXT: s_lshr_b32 s36, s1, 16
+; GFX11-NEXT: s_lshr_b32 s37, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s39, s0, 8
; GFX11-NEXT: s_lshr_b64 s[4:5], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[8:9], s[22:23], 24
@@ -40043,9 +40776,64 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b64 s[14:15], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[40:41], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, vcc_lo
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_3
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s48, -1
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr95
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s48, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -40100,30 +40888,30 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s91, s17, 8
; GFX11-NEXT: s_lshr_b32 s95, s16, 16
; GFX11-NEXT: s_lshr_b32 s94, s16, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
-; GFX11-NEXT: s_lshr_b32 s30, s3, 16
-; GFX11-NEXT: s_lshr_b32 s31, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s35, s2, 8
-; GFX11-NEXT: s_lshr_b32 s36, s1, 24
-; GFX11-NEXT: s_lshr_b32 s37, s1, 16
-; GFX11-NEXT: s_lshr_b32 s38, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s48, s0, 8
-; GFX11-NEXT: .LBB69_3: ; %end
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s3, 24
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 16
+; GFX11-NEXT: s_lshr_b32 s30, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s34, s2, 8
+; GFX11-NEXT: s_lshr_b32 s35, s1, 24
+; GFX11-NEXT: s_lshr_b32 s36, s1, 16
+; GFX11-NEXT: s_lshr_b32 s37, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s39, s0, 8
+; GFX11-NEXT: .LBB69_5: ; %end
; GFX11-NEXT: v_mov_b32_e32 v12, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v5, s34, s28, v12
-; GFX11-NEXT: v_readlane_b32 s34, v23, 0
+; GFX11-NEXT: v_readlane_b32 s48, v23, 6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v5, s31, s28, v12
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_perm_b32 v2, s39, s40, v12
-; GFX11-NEXT: v_perm_b32 v4, s37, s36, v12
-; GFX11-NEXT: v_perm_b32 v1, s0, s48, v12
-; GFX11-NEXT: v_perm_b32 v3, s1, s38, v12
-; GFX11-NEXT: v_perm_b32 v6, s2, s35, v12
+; GFX11-NEXT: v_perm_b32 v2, s38, s40, v12
+; GFX11-NEXT: v_perm_b32 v4, s36, s35, v12
+; GFX11-NEXT: v_perm_b32 v1, s0, s39, v12
+; GFX11-NEXT: v_perm_b32 v3, s1, s37, v12
+; GFX11-NEXT: v_perm_b32 v6, s2, s34, v12
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_perm_b32 v8, s30, vcc_hi, v12
+; GFX11-NEXT: v_perm_b32 v8, vcc_hi, vcc_lo, v12
; GFX11-NEXT: v_perm_b32 v9, s95, s14, v12
; GFX11-NEXT: v_perm_b32 v10, s17, s91, v12
; GFX11-NEXT: v_or_b32_e32 v1, v1, v2
@@ -40135,7 +40923,7 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: v_perm_b32 v8, s90, s12, v12
; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-NEXT: v_perm_b32 v7, s3, s31, v12
+; GFX11-NEXT: v_perm_b32 v7, s3, s30, v12
; GFX11-NEXT: v_perm_b32 v13, s18, s89, v12
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11-NEXT: v_or_b32_e32 v5, v6, v9
@@ -40185,67 +40973,17 @@ define inreg <64 x i8> @bitcast_v8i64_to_v64i8_scalar(<8 x i64> inreg %a, i32 in
; GFX11-NEXT: scratch_store_b128 v0, v[9:12], off offset:32
; GFX11-NEXT: scratch_store_b128 v0, v[13:16], off offset:48
; GFX11-NEXT: v_readlane_b32 s31, v23, 8
-; GFX11-NEXT: v_readlane_b32 s48, v23, 6
; GFX11-NEXT: v_readlane_b32 s39, v23, 5
; GFX11-NEXT: v_readlane_b32 s38, v23, 4
; GFX11-NEXT: v_readlane_b32 s37, v23, 3
; GFX11-NEXT: v_readlane_b32 s36, v23, 2
; GFX11-NEXT: v_readlane_b32 s35, v23, 1
+; GFX11-NEXT: v_readlane_b32 s34, v23, 0
; GFX11-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-NEXT: scratch_load_b32 v23, off, s32 ; 4-byte Folded Reload
; GFX11-NEXT: s_mov_b32 exec_lo, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_4:
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr31
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr95
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: s_branch .LBB69_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -42613,7 +43351,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
; SI-NEXT: v_or_b32_e32 v0, s4, v34
@@ -42762,8 +43500,17 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v8, s91
; SI-NEXT: v_mov_b32_e32 v9, s92
; SI-NEXT: v_mov_b32_e32 v10, s93
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -42974,12 +43721,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v8, s15
; SI-NEXT: v_mov_b32_e32 v9, s11
; SI-NEXT: v_mov_b32_e32 v10, s7
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v64i8_to_v8i64_scalar:
; VI: ; %bb.0:
@@ -43038,7 +43782,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -43130,8 +43874,17 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v19, v18, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -43285,12 +44038,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v64i8_to_v8i64_scalar:
; GFX9: ; %bb.0:
@@ -43349,7 +44099,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -43443,8 +44193,17 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX9-NEXT: v_perm_b32 v35, v19, v18, s4
; GFX9-NEXT: v_or_b32_e32 v15, v35, v15
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -43585,12 +44344,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v15
; GFX9-NEXT: v_add_u32_sdwa v16, v16, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8i64_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -43647,7 +44403,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -43719,9 +44475,17 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-TRUE16-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB71_3
+; GFX11-TRUE16-NEXT: .LBB71_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -43869,12 +44633,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-TRUE16-NEXT: .LBB71_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB71_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB71_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v8i64_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -43931,7 +44692,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -44003,9 +44764,17 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-FAKE16-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB71_3
+; GFX11-FAKE16-NEXT: .LBB71_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
@@ -44153,12 +44922,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-FAKE16-NEXT: .LBB71_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB71_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -44385,7 +45151,7 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB73_3
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s63, s5, 16
; SI-NEXT: s_lshr_b32 s62, s29, 16
@@ -44403,8 +45169,32 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB73_4
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -44429,26 +45219,8 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB73_5
-; SI-NEXT: .LBB73_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: s_branch .LBB73_2
-; SI-NEXT: .LBB73_4:
+; SI-NEXT: s_branch .LBB73_6
+; SI-NEXT: .LBB73_5:
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v11, s27
@@ -44481,7 +45253,7 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v18, s10
; SI-NEXT: v_mov_b32_e32 v17, s8
; SI-NEXT: v_mov_b32_e32 v16, s6
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
@@ -44544,10 +45316,18 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -44556,10 +45336,8 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: s_branch .LBB73_5
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
+; VI-NEXT: s_branch .LBB73_6
+; VI-NEXT: .LBB73_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -44576,7 +45354,7 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB73_5: ; %end
+; VI-NEXT: .LBB73_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -44597,10 +45375,18 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -44609,10 +45395,8 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: s_branch .LBB73_5
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: s_branch .LBB73_6
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -44629,7 +45413,7 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB73_5: ; %end
+; GFX9-NEXT: .LBB73_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -44647,11 +45431,16 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB73_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -44661,8 +45450,6 @@ define inreg <32 x i16> @bitcast_v8f64_to_v32i16_scalar(<8 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB73_3:
-; GFX11-NEXT: s_branch .LBB73_2
; GFX11-NEXT: .LBB73_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -45062,7 +45849,7 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s11, s12, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_4
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -45112,8 +45899,17 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s4, s9, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB75_3
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -45194,7 +45990,7 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: .LBB75_3: ; %end
+; SI-NEXT: .LBB75_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -45224,9 +46020,6 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB75_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB75_2
;
; VI-LABEL: bitcast_v32i16_to_v8f64_scalar:
; VI: ; %bb.0:
@@ -45235,10 +46028,18 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB75_4
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB75_3
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s6, 3
; VI-NEXT: s_and_b32 s4, s6, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -45319,7 +46120,7 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB75_3: ; %end
+; VI-NEXT: .LBB75_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -45337,8 +46138,6 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB75_4:
-; VI-NEXT: s_branch .LBB75_2
;
; GFX9-LABEL: bitcast_v32i16_to_v8f64_scalar:
; GFX9: ; %bb.0:
@@ -45352,10 +46151,18 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB75_4
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -45372,10 +46179,8 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB75_5
-; GFX9-NEXT: .LBB75_3:
-; GFX9-NEXT: s_branch .LBB75_2
-; GFX9-NEXT: .LBB75_4:
+; GFX9-NEXT: s_branch .LBB75_6
+; GFX9-NEXT: .LBB75_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -45392,7 +46197,7 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB75_5: ; %end
+; GFX9-NEXT: .LBB75_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -45410,11 +46215,16 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB75_4
-; GFX11-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB75_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -45432,8 +46242,6 @@ define inreg <8 x double> @bitcast_v32i16_to_v8f64_scalar(<32 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: s_branch .LBB75_2
; GFX11-NEXT: .LBB75_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -45670,7 +46478,7 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB77_3
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s63, s5, 16
; SI-NEXT: s_lshr_b32 s62, s29, 16
@@ -45688,8 +46496,32 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB77_4
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[46:47], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[46:47], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -45714,26 +46546,8 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
-; SI-NEXT: s_branch .LBB77_5
-; SI-NEXT: .LBB77_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: s_branch .LBB77_2
-; SI-NEXT: .LBB77_4:
+; SI-NEXT: s_branch .LBB77_6
+; SI-NEXT: .LBB77_5:
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: v_mov_b32_e32 v13, s29
; SI-NEXT: v_mov_b32_e32 v11, s27
@@ -45766,7 +46580,7 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s10
; SI-NEXT: v_mov_b32_e32 v17, s8
; SI-NEXT: v_mov_b32_e32 v16, s6
-; SI-NEXT: .LBB77_5: ; %end
+; SI-NEXT: .LBB77_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
@@ -45829,10 +46643,18 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -45841,10 +46663,8 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: s_branch .LBB77_5
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: s_branch .LBB77_6
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -45861,7 +46681,7 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB77_5: ; %end
+; VI-NEXT: .LBB77_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -45882,10 +46702,18 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -45894,10 +46722,8 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: s_branch .LBB77_5
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: s_branch .LBB77_6
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -45914,7 +46740,7 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB77_5: ; %end
+; GFX9-NEXT: .LBB77_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -45932,11 +46758,16 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB77_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -45946,8 +46777,6 @@ define inreg <32 x half> @bitcast_v8f64_to_v32f16_scalar(<8 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB77_3:
-; GFX11-NEXT: s_branch .LBB77_2
; GFX11-NEXT: .LBB77_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -46411,7 +47240,7 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s10, s8, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_3
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s63, 16
@@ -46461,8 +47290,17 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB79_4
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s62
@@ -46591,11 +47429,8 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v14, v15, v14
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: s_branch .LBB79_5
-; SI-NEXT: .LBB79_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
-; SI-NEXT: s_branch .LBB79_2
-; SI-NEXT: .LBB79_4:
+; SI-NEXT: s_branch .LBB79_6
+; SI-NEXT: .LBB79_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -46612,7 +47447,7 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v13, s49
; SI-NEXT: v_mov_b32_e32 v14, s50
; SI-NEXT: v_mov_b32_e32 v15, s51
-; SI-NEXT: .LBB79_5: ; %end
+; SI-NEXT: .LBB79_6: ; %end
; SI-NEXT: v_readlane_b32 s51, v18, 7
; SI-NEXT: v_readlane_b32 s50, v18, 6
; SI-NEXT: v_readlane_b32 s49, v18, 5
@@ -46639,10 +47474,18 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB79_3
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB79_4
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s31, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -46724,10 +47567,8 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v16, v16, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB79_5
-; VI-NEXT: .LBB79_3:
-; VI-NEXT: s_branch .LBB79_2
-; VI-NEXT: .LBB79_4:
+; VI-NEXT: s_branch .LBB79_6
+; VI-NEXT: .LBB79_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -46744,7 +47585,7 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB79_5: ; %end
+; VI-NEXT: .LBB79_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v17, 0
; VI-NEXT: v_readlane_b32 s31, v17, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -46765,10 +47606,18 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB79_4
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -46786,10 +47635,8 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB79_5
-; GFX9-NEXT: .LBB79_3:
-; GFX9-NEXT: s_branch .LBB79_2
-; GFX9-NEXT: .LBB79_4:
+; GFX9-NEXT: s_branch .LBB79_6
+; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -46806,7 +47653,7 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB79_5: ; %end
+; GFX9-NEXT: .LBB79_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -46824,11 +47671,16 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB79_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -46846,8 +47698,6 @@ define inreg <8 x double> @bitcast_v32f16_to_v8f64_scalar(<32 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_3:
-; GFX11-NEXT: s_branch .LBB79_2
; GFX11-NEXT: .LBB79_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -47156,7 +48006,7 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s79, s5, 0xffff0000
; SI-NEXT: s_lshl_b32 s78, s5, 16
@@ -47190,8 +48040,48 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; SI-NEXT: s_lshl_b32 s10, s17, 16
; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[19:20], s[22:23], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[24:25], 1.0
; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -47232,42 +48122,8 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v31
; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v30
; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; SI-NEXT: s_branch .LBB81_5
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: s_branch .LBB81_6
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v1, s79
; SI-NEXT: v_mov_b32_e32 v0, s78
; SI-NEXT: v_mov_b32_e32 v3, s77
@@ -47300,7 +48156,7 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v28, s10
; SI-NEXT: v_mov_b32_e32 v31, s9
; SI-NEXT: v_mov_b32_e32 v30, s8
-; SI-NEXT: .LBB81_5: ; %end
+; SI-NEXT: .LBB81_6: ; %end
; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
@@ -47379,10 +48235,18 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -47391,10 +48255,8 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: s_branch .LBB81_5
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: s_branch .LBB81_6
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -47411,7 +48273,7 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB81_5: ; %end
+; VI-NEXT: .LBB81_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v16, 0
; VI-NEXT: v_readlane_b32 s31, v16, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -47432,10 +48294,18 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[14:15], s[30:31], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
@@ -47444,10 +48314,8 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: s_branch .LBB81_5
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: s_branch .LBB81_6
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -47464,7 +48332,7 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB81_5: ; %end
+; GFX9-NEXT: .LBB81_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -47482,11 +48350,16 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
; GFX11-NEXT: v_add_f64 v[10:11], s[22:23], 1.0
@@ -47496,8 +48369,6 @@ define inreg <32 x bfloat> @bitcast_v8f64_to_v32bf16_scalar(<8 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[14:15], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[12:13], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -49020,7 +49891,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s44
; SI-NEXT: v_mul_f32_e64 v17, 1.0, s42
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
@@ -49056,8 +49927,17 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_mov_b32_e32 v20, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[17:18], 16
; SI-NEXT: v_mov_b32_e32 v16, v20
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -49154,7 +50034,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -49173,9 +50053,6 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v32bf16_to_v8f64_scalar:
; VI: ; %bb.0:
@@ -49189,10 +50066,18 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB83_3
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_4
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s30, 16
; VI-NEXT: v_add_f32_e32 v0, s4, v16
@@ -49490,10 +50375,8 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_lshrrev_b64 v[16:17], 16, v[17:18]
; VI-NEXT: v_mov_b32_e32 v13, v15
; VI-NEXT: v_mov_b32_e32 v15, v16
-; VI-NEXT: s_branch .LBB83_5
-; VI-NEXT: .LBB83_3:
-; VI-NEXT: s_branch .LBB83_2
-; VI-NEXT: .LBB83_4:
+; VI-NEXT: s_branch .LBB83_6
+; VI-NEXT: .LBB83_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -49510,7 +50393,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB83_5: ; %end
+; VI-NEXT: .LBB83_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -49531,10 +50414,18 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_4
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s4, s31, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -49841,10 +50732,8 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
-; GFX9-NEXT: s_branch .LBB83_5
-; GFX9-NEXT: .LBB83_3:
-; GFX9-NEXT: s_branch .LBB83_2
-; GFX9-NEXT: .LBB83_4:
+; GFX9-NEXT: s_branch .LBB83_6
+; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -49861,7 +50750,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB83_5: ; %end
+; GFX9-NEXT: .LBB83_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -49879,11 +50768,16 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-TRUE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s27, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s27, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -50193,8 +51087,6 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB83_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB83_2
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -50215,11 +51107,16 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-FAKE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s1, s27, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s27, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
@@ -50548,8 +51445,6 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v16, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v20, 16, v17
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB83_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB83_2
; GFX11-FAKE16-NEXT: .LBB83_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -52040,40 +52935,38 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_writelane_b32 v40, s83, 25
; SI-NEXT: v_writelane_b32 v40, s84, 26
; SI-NEXT: v_writelane_b32 v40, s85, 27
-; SI-NEXT: v_writelane_b32 v40, s86, 28
-; SI-NEXT: v_writelane_b32 v40, s87, 29
-; SI-NEXT: v_writelane_b32 v40, s30, 30
-; SI-NEXT: v_writelane_b32 v40, s31, 31
+; SI-NEXT: v_writelane_b32 v40, s30, 28
+; SI-NEXT: v_writelane_b32 v40, s31, 29
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: v_readfirstlane_b32 s5, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_cbranch_scc0 .LBB85_3
+; SI-NEXT: s_cbranch_scc0 .LBB85_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s48, s5, 24
-; SI-NEXT: s_lshr_b32 s49, s5, 16
-; SI-NEXT: s_lshr_b32 s50, s5, 8
-; SI-NEXT: s_lshr_b32 s51, s29, 24
-; SI-NEXT: s_lshr_b32 s52, s29, 16
-; SI-NEXT: s_lshr_b32 s53, s29, 8
-; SI-NEXT: s_lshr_b32 s54, s27, 24
-; SI-NEXT: s_lshr_b32 s55, s27, 16
-; SI-NEXT: s_lshr_b32 s64, s27, 8
-; SI-NEXT: s_lshr_b32 s65, s25, 24
-; SI-NEXT: s_lshr_b32 s66, s25, 16
-; SI-NEXT: s_lshr_b32 s67, s25, 8
-; SI-NEXT: s_lshr_b32 s68, s23, 24
-; SI-NEXT: s_lshr_b32 s69, s23, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 8
-; SI-NEXT: s_lshr_b32 s71, s21, 24
-; SI-NEXT: s_lshr_b32 s80, s21, 16
-; SI-NEXT: s_lshr_b32 s81, s21, 8
-; SI-NEXT: s_lshr_b32 s82, s19, 24
-; SI-NEXT: s_lshr_b32 s83, s19, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 8
-; SI-NEXT: s_lshr_b32 s85, s17, 24
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s17, 8
+; SI-NEXT: s_lshr_b32 s38, s5, 24
+; SI-NEXT: s_lshr_b32 s39, s5, 16
+; SI-NEXT: s_lshr_b32 s48, s5, 8
+; SI-NEXT: s_lshr_b32 s49, s29, 24
+; SI-NEXT: s_lshr_b32 s50, s29, 16
+; SI-NEXT: s_lshr_b32 s51, s29, 8
+; SI-NEXT: s_lshr_b32 s52, s27, 24
+; SI-NEXT: s_lshr_b32 s53, s27, 16
+; SI-NEXT: s_lshr_b32 s54, s27, 8
+; SI-NEXT: s_lshr_b32 s55, s25, 24
+; SI-NEXT: s_lshr_b32 s64, s25, 16
+; SI-NEXT: s_lshr_b32 s65, s25, 8
+; SI-NEXT: s_lshr_b32 s66, s23, 24
+; SI-NEXT: s_lshr_b32 s67, s23, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 8
+; SI-NEXT: s_lshr_b32 s69, s21, 24
+; SI-NEXT: s_lshr_b32 s70, s21, 16
+; SI-NEXT: s_lshr_b32 s71, s21, 8
+; SI-NEXT: s_lshr_b32 s80, s19, 24
+; SI-NEXT: s_lshr_b32 s81, s19, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 8
+; SI-NEXT: s_lshr_b32 s83, s17, 24
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s17, 8
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 8
@@ -52085,10 +52978,10 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[88:89], s[26:27], 8
; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
; SI-NEXT: s_lshr_b64 s[94:95], s[24:25], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[24:25], 8
-; SI-NEXT: s_lshr_b64 s[34:35], s[22:23], 24
-; SI-NEXT: s_lshr_b64 s[36:37], s[22:23], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[22:23], 8
+; SI-NEXT: s_lshr_b64 vcc, s[24:25], 8
+; SI-NEXT: s_lshr_b64 s[30:31], s[22:23], 24
+; SI-NEXT: s_lshr_b64 s[34:35], s[22:23], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[22:23], 8
; SI-NEXT: s_lshr_b64 s[8:9], s[20:21], 24
; SI-NEXT: s_lshr_b64 s[12:13], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[20:21], 8
@@ -52098,91 +52991,25 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[58:59], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[62:63], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB85_4
-; SI-NEXT: .LBB85_2: ; %cmp.true
-; SI-NEXT: v_add_f64 v[28:29], s[18:19], 1.0
-; SI-NEXT: v_add_f64 v[5:6], s[26:27], 1.0
-; SI-NEXT: v_add_f64 v[13:14], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[3:4], s[28:29], 1.0
-; SI-NEXT: v_lshr_b64 v[48:49], v[28:29], 24
-; SI-NEXT: v_add_f64 v[1:2], s[4:5], 1.0
-; SI-NEXT: v_add_f64 v[7:8], s[24:25], 1.0
-; SI-NEXT: v_add_f64 v[20:21], s[20:21], 1.0
-; SI-NEXT: v_add_f64 v[32:33], s[16:17], 1.0
-; SI-NEXT: v_lshr_b64 v[22:23], v[5:6], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[13:14], 16
-; SI-NEXT: v_lshr_b64 v[49:50], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 24
-; SI-NEXT: v_lshr_b64 v[23:24], v[5:6], 8
-; SI-NEXT: v_lshr_b64 v[35:36], v[13:14], 8
-; SI-NEXT: v_lshr_b64 v[50:51], v[28:29], 8
-; SI-NEXT: v_lshr_b64 v[9:10], v[1:2], 24
-; SI-NEXT: v_lshr_b64 v[16:17], v[3:4], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[7:8], 24
-; SI-NEXT: v_lshr_b64 v[36:37], v[20:21], 24
-; SI-NEXT: v_lshr_b64 v[51:52], v[32:33], 24
-; SI-NEXT: v_lshr_b64 v[10:11], v[1:2], 16
-; SI-NEXT: v_lshr_b64 v[17:18], v[3:4], 8
-; SI-NEXT: v_lshr_b64 v[25:26], v[7:8], 16
-; SI-NEXT: v_lshr_b64 v[37:38], v[20:21], 16
-; SI-NEXT: v_lshr_b64 v[52:53], v[32:33], 16
-; SI-NEXT: v_readfirstlane_b32 s17, v33
-; SI-NEXT: v_readfirstlane_b32 s19, v29
-; SI-NEXT: v_readfirstlane_b32 s21, v21
-; SI-NEXT: v_readfirstlane_b32 s23, v14
-; SI-NEXT: v_readfirstlane_b32 s25, v8
-; SI-NEXT: v_readfirstlane_b32 s27, v6
-; SI-NEXT: v_readfirstlane_b32 s29, v4
-; SI-NEXT: v_readfirstlane_b32 s5, v2
-; SI-NEXT: v_lshr_b64 v[11:12], v[1:2], 8
-; SI-NEXT: v_lshr_b64 v[18:19], v[5:6], 24
-; SI-NEXT: v_lshr_b64 v[26:27], v[7:8], 8
-; SI-NEXT: v_lshr_b64 v[30:31], v[13:14], 24
-; SI-NEXT: v_lshr_b64 v[38:39], v[20:21], 8
-; SI-NEXT: v_lshr_b64 v[53:54], v[32:33], 8
-; SI-NEXT: s_lshr_b32 s48, s5, 24
-; SI-NEXT: s_lshr_b32 s49, s5, 16
-; SI-NEXT: s_lshr_b32 s50, s5, 8
-; SI-NEXT: s_lshr_b32 s51, s29, 24
-; SI-NEXT: s_lshr_b32 s52, s29, 16
-; SI-NEXT: s_lshr_b32 s53, s29, 8
-; SI-NEXT: s_lshr_b32 s54, s27, 24
-; SI-NEXT: s_lshr_b32 s55, s27, 16
-; SI-NEXT: s_lshr_b32 s64, s27, 8
-; SI-NEXT: s_lshr_b32 s65, s25, 24
-; SI-NEXT: s_lshr_b32 s66, s25, 16
-; SI-NEXT: s_lshr_b32 s67, s25, 8
-; SI-NEXT: s_lshr_b32 s68, s23, 24
-; SI-NEXT: s_lshr_b32 s69, s23, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 8
-; SI-NEXT: s_lshr_b32 s71, s21, 24
-; SI-NEXT: s_lshr_b32 s80, s21, 16
-; SI-NEXT: s_lshr_b32 s81, s21, 8
-; SI-NEXT: s_lshr_b32 s82, s19, 24
-; SI-NEXT: s_lshr_b32 s83, s19, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 8
-; SI-NEXT: s_lshr_b32 s85, s17, 24
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s17, 8
-; SI-NEXT: s_branch .LBB85_5
-; SI-NEXT: .LBB85_3:
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB85_3
+; SI-NEXT: .LBB85_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr62
; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr87
-; SI-NEXT: ; implicit-def: $sgpr86
; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr83
; SI-NEXT: ; implicit-def: $sgpr56
; SI-NEXT: ; implicit-def: $sgpr44
; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr83
; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr81
-; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr71
; SI-NEXT: ; implicit-def: $sgpr70
; SI-NEXT: ; implicit-def: $sgpr69
@@ -52199,10 +53026,12 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: ; implicit-def: $sgpr50
; SI-NEXT: ; implicit-def: $sgpr49
; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr39
; SI-NEXT: ; implicit-def: $sgpr38
; SI-NEXT: ; implicit-def: $sgpr36
; SI-NEXT: ; implicit-def: $sgpr34
; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr88
@@ -52214,8 +53043,78 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB85_2
-; SI-NEXT: .LBB85_4:
+; SI-NEXT: .LBB85_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB85_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f64 v[28:29], s[18:19], 1.0
+; SI-NEXT: v_add_f64 v[5:6], s[26:27], 1.0
+; SI-NEXT: v_add_f64 v[13:14], s[22:23], 1.0
+; SI-NEXT: v_add_f64 v[3:4], s[28:29], 1.0
+; SI-NEXT: v_lshr_b64 v[48:49], v[28:29], 24
+; SI-NEXT: v_add_f64 v[1:2], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[7:8], s[24:25], 1.0
+; SI-NEXT: v_add_f64 v[20:21], s[20:21], 1.0
+; SI-NEXT: v_add_f64 v[32:33], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[22:23], v[5:6], 16
+; SI-NEXT: v_lshr_b64 v[34:35], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[28:29], 16
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 24
+; SI-NEXT: v_lshr_b64 v[23:24], v[5:6], 8
+; SI-NEXT: v_lshr_b64 v[35:36], v[13:14], 8
+; SI-NEXT: v_lshr_b64 v[50:51], v[28:29], 8
+; SI-NEXT: v_lshr_b64 v[9:10], v[1:2], 24
+; SI-NEXT: v_lshr_b64 v[16:17], v[3:4], 16
+; SI-NEXT: v_lshr_b64 v[24:25], v[7:8], 24
+; SI-NEXT: v_lshr_b64 v[36:37], v[20:21], 24
+; SI-NEXT: v_lshr_b64 v[51:52], v[32:33], 24
+; SI-NEXT: v_lshr_b64 v[10:11], v[1:2], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[3:4], 8
+; SI-NEXT: v_lshr_b64 v[25:26], v[7:8], 16
+; SI-NEXT: v_lshr_b64 v[37:38], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[32:33], 16
+; SI-NEXT: v_readfirstlane_b32 s17, v33
+; SI-NEXT: v_readfirstlane_b32 s19, v29
+; SI-NEXT: v_readfirstlane_b32 s21, v21
+; SI-NEXT: v_readfirstlane_b32 s23, v14
+; SI-NEXT: v_readfirstlane_b32 s25, v8
+; SI-NEXT: v_readfirstlane_b32 s27, v6
+; SI-NEXT: v_readfirstlane_b32 s29, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v2
+; SI-NEXT: v_lshr_b64 v[11:12], v[1:2], 8
+; SI-NEXT: v_lshr_b64 v[18:19], v[5:6], 24
+; SI-NEXT: v_lshr_b64 v[26:27], v[7:8], 8
+; SI-NEXT: v_lshr_b64 v[30:31], v[13:14], 24
+; SI-NEXT: v_lshr_b64 v[38:39], v[20:21], 8
+; SI-NEXT: v_lshr_b64 v[53:54], v[32:33], 8
+; SI-NEXT: s_lshr_b32 s38, s5, 24
+; SI-NEXT: s_lshr_b32 s39, s5, 16
+; SI-NEXT: s_lshr_b32 s48, s5, 8
+; SI-NEXT: s_lshr_b32 s49, s29, 24
+; SI-NEXT: s_lshr_b32 s50, s29, 16
+; SI-NEXT: s_lshr_b32 s51, s29, 8
+; SI-NEXT: s_lshr_b32 s52, s27, 24
+; SI-NEXT: s_lshr_b32 s53, s27, 16
+; SI-NEXT: s_lshr_b32 s54, s27, 8
+; SI-NEXT: s_lshr_b32 s55, s25, 24
+; SI-NEXT: s_lshr_b32 s64, s25, 16
+; SI-NEXT: s_lshr_b32 s65, s25, 8
+; SI-NEXT: s_lshr_b32 s66, s23, 24
+; SI-NEXT: s_lshr_b32 s67, s23, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 8
+; SI-NEXT: s_lshr_b32 s69, s21, 24
+; SI-NEXT: s_lshr_b32 s70, s21, 16
+; SI-NEXT: s_lshr_b32 s71, s21, 8
+; SI-NEXT: s_lshr_b32 s80, s19, 24
+; SI-NEXT: s_lshr_b32 s81, s19, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 8
+; SI-NEXT: s_lshr_b32 s83, s17, 24
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s17, 8
+; SI-NEXT: s_branch .LBB85_6
+; SI-NEXT: .LBB85_5:
; SI-NEXT: v_mov_b32_e32 v32, s16
; SI-NEXT: v_mov_b32_e32 v28, s18
; SI-NEXT: v_mov_b32_e32 v20, s20
@@ -52233,10 +53132,10 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v38, s40
; SI-NEXT: v_mov_b32_e32 v37, s12
; SI-NEXT: v_mov_b32_e32 v36, s8
-; SI-NEXT: v_mov_b32_e32 v35, s38
-; SI-NEXT: v_mov_b32_e32 v34, s36
-; SI-NEXT: v_mov_b32_e32 v30, s34
-; SI-NEXT: v_mov_b32_e32 v26, s30
+; SI-NEXT: v_mov_b32_e32 v35, s36
+; SI-NEXT: v_mov_b32_e32 v34, s34
+; SI-NEXT: v_mov_b32_e32 v30, s30
+; SI-NEXT: v_mov_b32_e32 v26, vcc_lo
; SI-NEXT: v_mov_b32_e32 v25, s94
; SI-NEXT: v_mov_b32_e32 v24, s92
; SI-NEXT: v_mov_b32_e32 v23, s88
@@ -52248,19 +53147,19 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v11, s14
; SI-NEXT: v_mov_b32_e32 v10, s10
; SI-NEXT: v_mov_b32_e32 v9, s6
-; SI-NEXT: .LBB85_5: ; %end
+; SI-NEXT: .LBB85_6: ; %end
; SI-NEXT: v_and_b32_e32 v2, 0xff, v32
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v53
; SI-NEXT: s_and_b32 s4, s17, 0xff
-; SI-NEXT: s_lshl_b32 s6, s87, 8
+; SI-NEXT: s_lshl_b32 s6, s85, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v52
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s86, 0xff
+; SI-NEXT: s_and_b32 s6, s84, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v51
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s85, 24
+; SI-NEXT: s_lshl_b32 s7, s83, 24
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52276,15 +53175,15 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v50
; SI-NEXT: s_and_b32 s4, s19, 0xff
-; SI-NEXT: s_lshl_b32 s6, s84, 8
+; SI-NEXT: s_lshl_b32 s6, s82, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v49
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s83, 0xff
+; SI-NEXT: s_and_b32 s6, s81, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v48
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s82, 24
+; SI-NEXT: s_lshl_b32 s7, s80, 24
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52301,15 +53200,15 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v38
; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s6, s81, 8
+; SI-NEXT: s_lshl_b32 s6, s71, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v37
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s80, 0xff
+; SI-NEXT: s_and_b32 s6, s70, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v36
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s71, 24
+; SI-NEXT: s_lshl_b32 s7, s69, 24
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52326,15 +53225,15 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v35
; SI-NEXT: s_and_b32 s4, s23, 0xff
-; SI-NEXT: s_lshl_b32 s6, s70, 8
+; SI-NEXT: s_lshl_b32 s6, s68, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v34
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s69, 0xff
+; SI-NEXT: s_and_b32 s6, s67, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v30
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s68, 24
+; SI-NEXT: s_lshl_b32 s7, s66, 24
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52351,15 +53250,15 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v26
; SI-NEXT: s_and_b32 s4, s25, 0xff
-; SI-NEXT: s_lshl_b32 s6, s67, 8
+; SI-NEXT: s_lshl_b32 s6, s65, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v25
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s66, 0xff
+; SI-NEXT: s_and_b32 s6, s64, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v24
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s65, 24
+; SI-NEXT: s_lshl_b32 s7, s55, 24
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52376,15 +53275,15 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v23
; SI-NEXT: s_and_b32 s4, s27, 0xff
-; SI-NEXT: s_lshl_b32 s6, s64, 8
+; SI-NEXT: s_lshl_b32 s6, s54, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v22
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s55, 0xff
+; SI-NEXT: s_and_b32 s6, s53, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v18
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s54, 24
+; SI-NEXT: s_lshl_b32 s7, s52, 24
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v5, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52400,16 +53299,16 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v17
; SI-NEXT: s_and_b32 s4, s29, 0xff
-; SI-NEXT: s_lshl_b32 s6, s53, 8
+; SI-NEXT: s_lshl_b32 s6, s51, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_and_b32_e32 v3, 0xff, v16
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s52, 0xff
+; SI-NEXT: s_and_b32 s6, s50, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v15
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s51, 24
+; SI-NEXT: s_lshl_b32 s7, s49, 24
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52425,16 +53324,16 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v11
; SI-NEXT: s_and_b32 s4, s5, 0xff
-; SI-NEXT: s_lshl_b32 s5, s50, 8
+; SI-NEXT: s_lshl_b32 s5, s48, 8
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_and_b32_e32 v2, 0xff, v10
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s49, 0xff
+; SI-NEXT: s_and_b32 s5, s39, 0xff
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v9
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s48, 24
+; SI-NEXT: s_lshl_b32 s6, s38, 24
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v2, v3, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
@@ -52446,11 +53345,9 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; SI-NEXT: v_add_i32_e32 v0, vcc, 60, v0
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: v_readlane_b32 s30, v40, 30
+; SI-NEXT: v_readlane_b32 s30, v40, 28
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; SI-NEXT: v_readlane_b32 s31, v40, 31
-; SI-NEXT: v_readlane_b32 s87, v40, 29
-; SI-NEXT: v_readlane_b32 s86, v40, 28
+; SI-NEXT: v_readlane_b32 s31, v40, 29
; SI-NEXT: v_readlane_b32 s85, v40, 27
; SI-NEXT: v_readlane_b32 s84, v40, 26
; SI-NEXT: v_readlane_b32 s83, v40, 25
@@ -52522,56 +53419,54 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_writelane_b32 v63, s55, 13
; VI-NEXT: v_writelane_b32 v63, s64, 14
; VI-NEXT: v_writelane_b32 v63, s65, 15
-; VI-NEXT: v_writelane_b32 v63, s66, 16
-; VI-NEXT: v_writelane_b32 v63, s67, 17
-; VI-NEXT: v_writelane_b32 v63, s30, 18
-; VI-NEXT: v_writelane_b32 v63, s31, 19
+; VI-NEXT: v_writelane_b32 v63, s30, 16
+; VI-NEXT: v_writelane_b32 v63, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB85_3
+; VI-NEXT: s_cbranch_scc0 .LBB85_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_lshr_b32 s35, s5, 24
-; VI-NEXT: s_lshr_b32 s34, s5, 16
-; VI-NEXT: s_lshr_b32 s31, s5, 8
-; VI-NEXT: s_lshr_b32 s37, s4, 16
-; VI-NEXT: s_lshr_b32 s36, s4, 8
-; VI-NEXT: s_lshr_b32 s30, s29, 24
+; VI-NEXT: s_lshr_b32 s31, s5, 24
+; VI-NEXT: s_lshr_b32 s30, s5, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s5, 8
+; VI-NEXT: s_lshr_b32 s35, s4, 16
+; VI-NEXT: s_lshr_b32 s34, s4, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s29, 24
; VI-NEXT: s_lshr_b32 s91, s29, 16
; VI-NEXT: s_lshr_b32 s90, s29, 8
-; VI-NEXT: s_lshr_b32 s39, s28, 16
-; VI-NEXT: s_lshr_b32 s38, s28, 8
+; VI-NEXT: s_lshr_b32 s37, s28, 16
+; VI-NEXT: s_lshr_b32 s36, s28, 8
; VI-NEXT: s_lshr_b32 s89, s27, 24
; VI-NEXT: s_lshr_b32 s88, s27, 16
; VI-NEXT: s_lshr_b32 s79, s27, 8
-; VI-NEXT: s_lshr_b32 s49, s26, 16
-; VI-NEXT: s_lshr_b32 s48, s26, 8
+; VI-NEXT: s_lshr_b32 s39, s26, 16
+; VI-NEXT: s_lshr_b32 s38, s26, 8
; VI-NEXT: s_lshr_b32 s78, s25, 24
; VI-NEXT: s_lshr_b32 s77, s25, 16
; VI-NEXT: s_lshr_b32 s76, s25, 8
-; VI-NEXT: s_lshr_b32 s51, s24, 16
-; VI-NEXT: s_lshr_b32 s50, s24, 8
+; VI-NEXT: s_lshr_b32 s49, s24, 16
+; VI-NEXT: s_lshr_b32 s48, s24, 8
; VI-NEXT: s_lshr_b32 s75, s23, 24
; VI-NEXT: s_lshr_b32 s74, s23, 16
; VI-NEXT: s_lshr_b32 s73, s23, 8
-; VI-NEXT: s_lshr_b32 s53, s22, 16
-; VI-NEXT: s_lshr_b32 s52, s22, 8
+; VI-NEXT: s_lshr_b32 s51, s22, 16
+; VI-NEXT: s_lshr_b32 s50, s22, 8
; VI-NEXT: s_lshr_b32 s72, s21, 24
; VI-NEXT: s_lshr_b32 s63, s21, 16
; VI-NEXT: s_lshr_b32 s62, s21, 8
-; VI-NEXT: s_lshr_b32 s55, s20, 16
-; VI-NEXT: s_lshr_b32 s54, s20, 8
+; VI-NEXT: s_lshr_b32 s53, s20, 16
+; VI-NEXT: s_lshr_b32 s52, s20, 8
; VI-NEXT: s_lshr_b32 s61, s19, 24
; VI-NEXT: s_lshr_b32 s60, s19, 16
; VI-NEXT: s_lshr_b32 s59, s19, 8
-; VI-NEXT: s_lshr_b32 s65, s18, 16
-; VI-NEXT: s_lshr_b32 s64, s18, 8
+; VI-NEXT: s_lshr_b32 s55, s18, 16
+; VI-NEXT: s_lshr_b32 s54, s18, 8
; VI-NEXT: s_lshr_b32 s58, s17, 24
; VI-NEXT: s_lshr_b32 s57, s17, 16
; VI-NEXT: s_lshr_b32 s56, s17, 8
-; VI-NEXT: s_lshr_b32 s67, s16, 16
-; VI-NEXT: s_lshr_b32 s66, s16, 8
+; VI-NEXT: s_lshr_b32 s65, s16, 16
+; VI-NEXT: s_lshr_b32 s64, s16, 8
; VI-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -52580,8 +53475,64 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB85_4
-; VI-NEXT: .LBB85_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB85_3
+; VI-NEXT: .LBB85_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: .LBB85_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB85_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[9:10], s[4:5], 1.0
; VI-NEXT: v_add_f64 v[1:2], s[28:29], 1.0
; VI-NEXT: v_add_f64 v[5:6], s[24:25], 1.0
@@ -52644,59 +53595,9 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v18, 8, v16
; VI-NEXT: v_lshrrev_b32_e32 v57, 16, v15
; VI-NEXT: v_lshrrev_b32_e32 v58, 8, v15
-; VI-NEXT: s_branch .LBB85_5
-; VI-NEXT: .LBB85_3:
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: s_branch .LBB85_2
-; VI-NEXT: .LBB85_4:
-; VI-NEXT: v_mov_b32_e32 v26, s35
+; VI-NEXT: s_branch .LBB85_6
+; VI-NEXT: .LBB85_5:
+; VI-NEXT: v_mov_b32_e32 v26, s31
; VI-NEXT: v_mov_b32_e32 v25, s40
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
@@ -52717,25 +53618,25 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v12, s21
; VI-NEXT: v_mov_b32_e32 v14, s19
; VI-NEXT: v_mov_b32_e32 v16, s17
-; VI-NEXT: v_mov_b32_e32 v57, s67
-; VI-NEXT: v_mov_b32_e32 v58, s66
-; VI-NEXT: v_mov_b32_e32 v44, s65
-; VI-NEXT: v_mov_b32_e32 v45, s64
-; VI-NEXT: v_mov_b32_e32 v55, s55
-; VI-NEXT: v_mov_b32_e32 v40, s54
-; VI-NEXT: v_mov_b32_e32 v50, s53
-; VI-NEXT: v_mov_b32_e32 v52, s52
-; VI-NEXT: v_mov_b32_e32 v39, s51
-; VI-NEXT: v_mov_b32_e32 v49, s50
-; VI-NEXT: v_mov_b32_e32 v35, s49
-; VI-NEXT: v_mov_b32_e32 v36, s48
-; VI-NEXT: v_mov_b32_e32 v30, s39
-; VI-NEXT: v_mov_b32_e32 v31, s38
-; VI-NEXT: v_mov_b32_e32 v19, s37
-; VI-NEXT: v_mov_b32_e32 v28, s36
-; VI-NEXT: v_mov_b32_e32 v27, s34
-; VI-NEXT: v_mov_b32_e32 v29, s31
-; VI-NEXT: v_mov_b32_e32 v32, s30
+; VI-NEXT: v_mov_b32_e32 v57, s65
+; VI-NEXT: v_mov_b32_e32 v58, s64
+; VI-NEXT: v_mov_b32_e32 v44, s55
+; VI-NEXT: v_mov_b32_e32 v45, s54
+; VI-NEXT: v_mov_b32_e32 v55, s53
+; VI-NEXT: v_mov_b32_e32 v40, s52
+; VI-NEXT: v_mov_b32_e32 v50, s51
+; VI-NEXT: v_mov_b32_e32 v52, s50
+; VI-NEXT: v_mov_b32_e32 v39, s49
+; VI-NEXT: v_mov_b32_e32 v49, s48
+; VI-NEXT: v_mov_b32_e32 v35, s39
+; VI-NEXT: v_mov_b32_e32 v36, s38
+; VI-NEXT: v_mov_b32_e32 v30, s37
+; VI-NEXT: v_mov_b32_e32 v31, s36
+; VI-NEXT: v_mov_b32_e32 v19, s35
+; VI-NEXT: v_mov_b32_e32 v28, s34
+; VI-NEXT: v_mov_b32_e32 v27, s30
+; VI-NEXT: v_mov_b32_e32 v29, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v32, vcc_lo
; VI-NEXT: v_mov_b32_e32 v33, s91
; VI-NEXT: v_mov_b32_e32 v34, s90
; VI-NEXT: v_mov_b32_e32 v37, s89
@@ -52766,8 +53667,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v25, s44
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v25, s35
-; VI-NEXT: .LBB85_5: ; %end
+; VI-NEXT: v_mov_b32_e32 v25, s31
+; VI-NEXT: .LBB85_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v24, v57, v24, s4
; VI-NEXT: v_perm_b32 v15, v15, v58, s4
@@ -52832,10 +53733,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v3, v3, v36, s4
; VI-NEXT: v_perm_b32 v1, v1, v31, s4
-; VI-NEXT: v_readlane_b32 s30, v63, 18
-; VI-NEXT: v_readlane_b32 s31, v63, 19
-; VI-NEXT: v_readlane_b32 s67, v63, 17
-; VI-NEXT: v_readlane_b32 s66, v63, 16
+; VI-NEXT: v_readlane_b32 s30, v63, 16
+; VI-NEXT: v_readlane_b32 s31, v63, 17
; VI-NEXT: v_readlane_b32 s65, v63, 15
; VI-NEXT: v_readlane_b32 s64, v63, 14
; VI-NEXT: v_readlane_b32 s55, v63, 13
@@ -52947,56 +53846,54 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_writelane_b32 v63, s51, 9
; GFX9-NEXT: v_writelane_b32 v63, s52, 10
; GFX9-NEXT: v_writelane_b32 v63, s53, 11
-; GFX9-NEXT: v_writelane_b32 v63, s54, 12
-; GFX9-NEXT: v_writelane_b32 v63, s55, 13
-; GFX9-NEXT: v_writelane_b32 v63, s30, 14
-; GFX9-NEXT: v_writelane_b32 v63, s31, 15
+; GFX9-NEXT: v_writelane_b32 v63, s30, 12
+; GFX9-NEXT: v_writelane_b32 v63, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB85_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s95, s5, 24
; GFX9-NEXT: s_lshr_b32 s94, s5, 16
; GFX9-NEXT: s_lshr_b32 s93, s5, 8
-; GFX9-NEXT: s_lshr_b32 s31, s4, 16
-; GFX9-NEXT: s_lshr_b32 s30, s4, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s4, 16
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s4, 8
; GFX9-NEXT: s_lshr_b32 s92, s29, 24
; GFX9-NEXT: s_lshr_b32 s91, s29, 16
; GFX9-NEXT: s_lshr_b32 s90, s29, 8
-; GFX9-NEXT: s_lshr_b32 s35, s28, 16
-; GFX9-NEXT: s_lshr_b32 s34, s28, 8
+; GFX9-NEXT: s_lshr_b32 s31, s28, 16
+; GFX9-NEXT: s_lshr_b32 s30, s28, 8
; GFX9-NEXT: s_lshr_b32 s89, s27, 24
; GFX9-NEXT: s_lshr_b32 s88, s27, 16
; GFX9-NEXT: s_lshr_b32 s79, s27, 8
-; GFX9-NEXT: s_lshr_b32 s37, s26, 16
-; GFX9-NEXT: s_lshr_b32 s36, s26, 8
+; GFX9-NEXT: s_lshr_b32 s35, s26, 16
+; GFX9-NEXT: s_lshr_b32 s34, s26, 8
; GFX9-NEXT: s_lshr_b32 s78, s25, 24
; GFX9-NEXT: s_lshr_b32 s77, s25, 16
; GFX9-NEXT: s_lshr_b32 s76, s25, 8
-; GFX9-NEXT: s_lshr_b32 s39, s24, 16
-; GFX9-NEXT: s_lshr_b32 s38, s24, 8
+; GFX9-NEXT: s_lshr_b32 s37, s24, 16
+; GFX9-NEXT: s_lshr_b32 s36, s24, 8
; GFX9-NEXT: s_lshr_b32 s75, s23, 24
; GFX9-NEXT: s_lshr_b32 s74, s23, 16
; GFX9-NEXT: s_lshr_b32 s73, s23, 8
-; GFX9-NEXT: s_lshr_b32 s49, s22, 16
-; GFX9-NEXT: s_lshr_b32 s48, s22, 8
+; GFX9-NEXT: s_lshr_b32 s39, s22, 16
+; GFX9-NEXT: s_lshr_b32 s38, s22, 8
; GFX9-NEXT: s_lshr_b32 s72, s21, 24
; GFX9-NEXT: s_lshr_b32 s63, s21, 16
; GFX9-NEXT: s_lshr_b32 s62, s21, 8
-; GFX9-NEXT: s_lshr_b32 s51, s20, 16
-; GFX9-NEXT: s_lshr_b32 s50, s20, 8
+; GFX9-NEXT: s_lshr_b32 s49, s20, 16
+; GFX9-NEXT: s_lshr_b32 s48, s20, 8
; GFX9-NEXT: s_lshr_b32 s61, s19, 24
; GFX9-NEXT: s_lshr_b32 s60, s19, 16
; GFX9-NEXT: s_lshr_b32 s59, s19, 8
-; GFX9-NEXT: s_lshr_b32 s53, s18, 16
-; GFX9-NEXT: s_lshr_b32 s52, s18, 8
+; GFX9-NEXT: s_lshr_b32 s51, s18, 16
+; GFX9-NEXT: s_lshr_b32 s50, s18, 8
; GFX9-NEXT: s_lshr_b32 s58, s17, 24
; GFX9-NEXT: s_lshr_b32 s57, s17, 16
; GFX9-NEXT: s_lshr_b32 s56, s17, 8
-; GFX9-NEXT: s_lshr_b32 s55, s16, 16
-; GFX9-NEXT: s_lshr_b32 s54, s16, 8
+; GFX9-NEXT: s_lshr_b32 s53, s16, 16
+; GFX9-NEXT: s_lshr_b32 s52, s16, 8
; GFX9-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -53005,8 +53902,64 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB85_4
-; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB85_3
+; GFX9-NEXT: .LBB85_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: .LBB85_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[11:12], s[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[1:2], s[28:29], 1.0
; GFX9-NEXT: v_add_f64 v[5:6], s[24:25], 1.0
@@ -53072,59 +54025,9 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v18, 8, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v57, 16, v15
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 8, v15
-; GFX9-NEXT: s_branch .LBB85_5
-; GFX9-NEXT: .LBB85_3:
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: s_branch .LBB85_2
-; GFX9-NEXT: .LBB85_4:
-; GFX9-NEXT: v_mov_b32_e32 v26, s31
+; GFX9-NEXT: s_branch .LBB85_6
+; GFX9-NEXT: .LBB85_5:
+; GFX9-NEXT: v_mov_b32_e32 v26, vcc_hi
; GFX9-NEXT: v_mov_b32_e32 v25, s40
; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -53146,21 +54049,21 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s21
; GFX9-NEXT: v_mov_b32_e32 v14, s19
; GFX9-NEXT: v_mov_b32_e32 v16, s17
-; GFX9-NEXT: v_mov_b32_e32 v57, s55
-; GFX9-NEXT: v_mov_b32_e32 v59, s54
-; GFX9-NEXT: v_mov_b32_e32 v56, s53
-; GFX9-NEXT: v_mov_b32_e32 v47, s52
-; GFX9-NEXT: v_mov_b32_e32 v42, s51
-; GFX9-NEXT: v_mov_b32_e32 v43, s50
-; GFX9-NEXT: v_mov_b32_e32 v50, s49
-; GFX9-NEXT: v_mov_b32_e32 v51, s48
-; GFX9-NEXT: v_mov_b32_e32 v38, s39
-; GFX9-NEXT: v_mov_b32_e32 v48, s38
-; GFX9-NEXT: v_mov_b32_e32 v35, s37
-; GFX9-NEXT: v_mov_b32_e32 v36, s36
-; GFX9-NEXT: v_mov_b32_e32 v30, s35
-; GFX9-NEXT: v_mov_b32_e32 v31, s34
-; GFX9-NEXT: v_mov_b32_e32 v27, s30
+; GFX9-NEXT: v_mov_b32_e32 v57, s53
+; GFX9-NEXT: v_mov_b32_e32 v59, s52
+; GFX9-NEXT: v_mov_b32_e32 v56, s51
+; GFX9-NEXT: v_mov_b32_e32 v47, s50
+; GFX9-NEXT: v_mov_b32_e32 v42, s49
+; GFX9-NEXT: v_mov_b32_e32 v43, s48
+; GFX9-NEXT: v_mov_b32_e32 v50, s39
+; GFX9-NEXT: v_mov_b32_e32 v51, s38
+; GFX9-NEXT: v_mov_b32_e32 v38, s37
+; GFX9-NEXT: v_mov_b32_e32 v48, s36
+; GFX9-NEXT: v_mov_b32_e32 v35, s35
+; GFX9-NEXT: v_mov_b32_e32 v36, s34
+; GFX9-NEXT: v_mov_b32_e32 v30, s31
+; GFX9-NEXT: v_mov_b32_e32 v31, s30
+; GFX9-NEXT: v_mov_b32_e32 v27, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v19, s95
; GFX9-NEXT: v_mov_b32_e32 v28, s94
; GFX9-NEXT: v_mov_b32_e32 v29, s93
@@ -53197,7 +54100,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: .LBB85_5: ; %end
+; GFX9-NEXT: .LBB85_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v24, v57, v24, s4
; GFX9-NEXT: v_perm_b32 v15, v15, v59, s4
@@ -53254,10 +54157,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v36, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v31, s4
-; GFX9-NEXT: v_readlane_b32 s30, v63, 14
-; GFX9-NEXT: v_readlane_b32 s31, v63, 15
-; GFX9-NEXT: v_readlane_b32 s55, v63, 13
-; GFX9-NEXT: v_readlane_b32 s54, v63, 12
+; GFX9-NEXT: v_readlane_b32 s30, v63, 12
+; GFX9-NEXT: v_readlane_b32 s31, v63, 13
; GFX9-NEXT: v_readlane_b32 s53, v63, 11
; GFX9-NEXT: v_readlane_b32 s52, v63, 10
; GFX9-NEXT: v_readlane_b32 s51, v63, 9
@@ -53340,12 +54241,11 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s90, s27, 24
; GFX11-NEXT: s_lshr_b32 s89, s27, 16
@@ -53360,33 +54260,33 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s76, s23, 24
; GFX11-NEXT: s_lshr_b32 s75, s23, 16
; GFX11-NEXT: s_lshr_b32 s74, s23, 8
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s22, 16
; GFX11-NEXT: s_lshr_b32 s95, s22, 8
; GFX11-NEXT: s_lshr_b32 s73, s21, 24
; GFX11-NEXT: s_lshr_b32 s72, s21, 16
; GFX11-NEXT: s_lshr_b32 s63, s21, 8
-; GFX11-NEXT: s_lshr_b32 s31, s20, 16
-; GFX11-NEXT: s_lshr_b32 s30, s20, 8
+; GFX11-NEXT: s_lshr_b32 s30, s20, 16
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s20, 8
; GFX11-NEXT: s_lshr_b32 s62, s19, 24
; GFX11-NEXT: s_lshr_b32 s61, s19, 16
; GFX11-NEXT: s_lshr_b32 s60, s19, 8
-; GFX11-NEXT: s_lshr_b32 s35, s18, 16
-; GFX11-NEXT: s_lshr_b32 s34, s18, 8
+; GFX11-NEXT: s_lshr_b32 s34, s18, 16
+; GFX11-NEXT: s_lshr_b32 s31, s18, 8
; GFX11-NEXT: s_lshr_b32 s59, s17, 24
; GFX11-NEXT: s_lshr_b32 s58, s17, 16
; GFX11-NEXT: s_lshr_b32 s57, s17, 8
-; GFX11-NEXT: s_lshr_b32 s37, s16, 16
-; GFX11-NEXT: s_lshr_b32 s36, s16, 8
+; GFX11-NEXT: s_lshr_b32 s36, s16, 16
+; GFX11-NEXT: s_lshr_b32 s35, s16, 8
; GFX11-NEXT: s_lshr_b32 s56, s3, 24
; GFX11-NEXT: s_lshr_b32 s47, s3, 16
; GFX11-NEXT: s_lshr_b32 s46, s3, 8
-; GFX11-NEXT: s_lshr_b32 s39, s2, 16
-; GFX11-NEXT: s_lshr_b32 s38, s2, 8
+; GFX11-NEXT: s_lshr_b32 s38, s2, 16
+; GFX11-NEXT: s_lshr_b32 s37, s2, 8
; GFX11-NEXT: s_lshr_b32 s45, s1, 24
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s43, s1, 8
-; GFX11-NEXT: s_lshr_b32 s49, s0, 16
-; GFX11-NEXT: s_lshr_b32 s48, s0, 8
+; GFX11-NEXT: s_lshr_b32 s48, s0, 16
+; GFX11-NEXT: s_lshr_b32 s39, s0, 8
; GFX11-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[14:15], s[22:23], 24
@@ -53395,9 +54295,64 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
-; GFX11-NEXT: .LBB85_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB85_3
+; GFX11-NEXT: .LBB85_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr95
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: .LBB85_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[7:8], s[20:21], 1.0
; GFX11-NEXT: v_add_f64 v[11:12], s[18:19], 1.0
; GFX11-NEXT: v_add_f64 v[15:16], s[16:17], 1.0
@@ -53454,58 +54409,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v86, 8, v22
; GFX11-NEXT: v_lshrrev_b32_e32 v82, 16, v21
; GFX11-NEXT: v_lshrrev_b32_e32 v81, 8, v21
-; GFX11-NEXT: s_branch .LBB85_5
-; GFX11-NEXT: .LBB85_3:
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $sgpr31
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr95
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: s_branch .LBB85_2
-; GFX11-NEXT: .LBB85_4:
+; GFX11-NEXT: s_branch .LBB85_6
+; GFX11-NEXT: .LBB85_5:
; GFX11-NEXT: v_dual_mov_b32 v21, s0 :: v_dual_mov_b32 v2, s27
; GFX11-NEXT: v_dual_mov_b32 v17, s2 :: v_dual_mov_b32 v4, s25
; GFX11-NEXT: v_dual_mov_b32 v15, s16 :: v_dual_mov_b32 v6, s23
@@ -53514,13 +54419,13 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v5, s22 :: v_dual_mov_b32 v16, s17
; GFX11-NEXT: v_dual_mov_b32 v3, s24 :: v_dual_mov_b32 v18, s3
; GFX11-NEXT: v_dual_mov_b32 v1, s26 :: v_dual_mov_b32 v22, s1
-; GFX11-NEXT: v_dual_mov_b32 v82, s49 :: v_dual_mov_b32 v81, s48
-; GFX11-NEXT: v_dual_mov_b32 v69, s39 :: v_dual_mov_b32 v68, s38
-; GFX11-NEXT: v_dual_mov_b32 v64, s37 :: v_dual_mov_b32 v55, s36
-; GFX11-NEXT: v_dual_mov_b32 v54, s35 :: v_dual_mov_b32 v53, s34
-; GFX11-NEXT: v_dual_mov_b32 v49, s31 :: v_dual_mov_b32 v30, s93
-; GFX11-NEXT: v_dual_mov_b32 v39, s30 :: v_dual_mov_b32 v14, s92
-; GFX11-NEXT: v_dual_mov_b32 v37, vcc_hi :: v_dual_mov_b32 v10, s91
+; GFX11-NEXT: v_dual_mov_b32 v82, s48 :: v_dual_mov_b32 v81, s39
+; GFX11-NEXT: v_dual_mov_b32 v69, s38 :: v_dual_mov_b32 v68, s37
+; GFX11-NEXT: v_dual_mov_b32 v64, s36 :: v_dual_mov_b32 v55, s35
+; GFX11-NEXT: v_dual_mov_b32 v54, s34 :: v_dual_mov_b32 v53, s31
+; GFX11-NEXT: v_dual_mov_b32 v49, s30 :: v_dual_mov_b32 v30, s93
+; GFX11-NEXT: v_dual_mov_b32 v39, vcc_hi :: v_dual_mov_b32 v14, s92
+; GFX11-NEXT: v_dual_mov_b32 v37, vcc_lo :: v_dual_mov_b32 v10, s91
; GFX11-NEXT: v_dual_mov_b32 v35, s95 :: v_dual_mov_b32 v26, s6
; GFX11-NEXT: v_dual_mov_b32 v31, s94 :: v_dual_mov_b32 v24, s10
; GFX11-NEXT: v_dual_mov_b32 v27, s4 :: v_dual_mov_b32 v28, s90
@@ -53538,7 +54443,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v84, s56 :: v_dual_mov_b32 v85, s47
; GFX11-NEXT: v_dual_mov_b32 v83, s46 :: v_dual_mov_b32 v96, s44
; GFX11-NEXT: v_dual_mov_b32 v87, s45 :: v_dual_mov_b32 v86, s43
-; GFX11-NEXT: .LBB85_5: ; %end
+; GFX11-NEXT: .LBB85_6: ; %end
; GFX11-NEXT: v_perm_b32 v26, v69, v26, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_perm_b32 v27, v82, v27, 0xc0c0004
@@ -53604,14 +54509,13 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v2, v4, v8
; GFX11-NEXT: v_or_b32_e32 v3, v10, v9
; GFX11-NEXT: v_or_b32_e32 v4, v12, v11
-; GFX11-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: scratch_store_b128 v0, v[84:87], off
; GFX11-NEXT: scratch_store_b128 v0, v[15:18], off offset:16
; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:32
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:48
-; GFX11-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-NEXT: v_readlane_b32 s38, v40, 4
@@ -55991,7 +56895,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
-; SI-NEXT: s_cbranch_scc0 .LBB87_4
+; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
; SI-NEXT: v_or_b32_e32 v0, s4, v34
@@ -56140,8 +57044,17 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s91
; SI-NEXT: v_mov_b32_e32 v9, s92
; SI-NEXT: v_mov_b32_e32 v10, s93
-; SI-NEXT: s_cbranch_execnz .LBB87_3
-; SI-NEXT: .LBB87_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB87_3
+; SI-NEXT: .LBB87_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; SI-NEXT: .LBB87_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB87_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -56352,12 +57265,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v8, s15
; SI-NEXT: v_mov_b32_e32 v9, s11
; SI-NEXT: v_mov_b32_e32 v10, s7
-; SI-NEXT: .LBB87_3: ; %end
+; SI-NEXT: .LBB87_5: ; %end
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB87_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; SI-NEXT: s_branch .LBB87_2
;
; VI-LABEL: bitcast_v64i8_to_v8f64_scalar:
; VI: ; %bb.0:
@@ -56416,7 +57326,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB87_4
+; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -56508,8 +57418,17 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v19, v18, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB87_3
-; VI-NEXT: .LBB87_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB87_3
+; VI-NEXT: .LBB87_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB87_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB87_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -56663,12 +57582,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB87_3: ; %end
+; VI-NEXT: .LBB87_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB87_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB87_2
;
; GFX9-LABEL: bitcast_v64i8_to_v8f64_scalar:
; GFX9: ; %bb.0:
@@ -56727,7 +57643,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -56821,8 +57737,17 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX9-NEXT: v_perm_b32 v35, v19, v18, s4
; GFX9-NEXT: v_or_b32_e32 v15, v35, v15
-; GFX9-NEXT: s_cbranch_execnz .LBB87_3
-; GFX9-NEXT: .LBB87_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB87_3
+; GFX9-NEXT: .LBB87_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB87_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -56963,12 +57888,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v15, 0x300, v15
; GFX9-NEXT: v_add_u32_sdwa v16, v16, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB87_3: ; %end
+; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB87_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB87_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v8f64_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -57025,7 +57947,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -57097,9 +58019,17 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB87_3
-; GFX11-TRUE16-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB87_3
+; GFX11-TRUE16-NEXT: .LBB87_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -57247,12 +58177,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-TRUE16-NEXT: .LBB87_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB87_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB87_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB87_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v8f64_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -57309,7 +58236,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
@@ -57381,9 +58308,17 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v17, v15
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v19, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v20, v18
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB87_3
-; GFX11-FAKE16-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB87_3
+; GFX11-FAKE16-NEXT: .LBB87_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
@@ -57531,12 +58466,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v14, v16
; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, v17, v18
; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, v19, v20
-; GFX11-FAKE16-NEXT: .LBB87_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB87_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB87_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB87_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -58014,31 +58946,29 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v16, s35, 1
; SI-NEXT: v_writelane_b32 v16, s36, 2
; SI-NEXT: v_writelane_b32 v16, s37, 3
-; SI-NEXT: v_writelane_b32 v16, s38, 4
-; SI-NEXT: v_writelane_b32 v16, s39, 5
-; SI-NEXT: v_writelane_b32 v16, s30, 6
-; SI-NEXT: v_writelane_b32 v16, s31, 7
-; SI-NEXT: v_readfirstlane_b32 s37, v1
-; SI-NEXT: v_readfirstlane_b32 s39, v0
+; SI-NEXT: v_writelane_b32 v16, s30, 4
+; SI-NEXT: v_writelane_b32 v16, s31, 5
+; SI-NEXT: v_readfirstlane_b32 s35, v1
+; SI-NEXT: v_readfirstlane_b32 s37, v0
; SI-NEXT: s_lshr_b32 s92, s29, 16
-; SI-NEXT: s_lshr_b32 s36, s28, 16
+; SI-NEXT: s_lshr_b32 s34, s28, 16
; SI-NEXT: s_lshr_b32 s91, s27, 16
-; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s31, s26, 16
; SI-NEXT: s_lshr_b32 s90, s25, 16
-; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s30, s24, 16
; SI-NEXT: s_lshr_b32 s89, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
; SI-NEXT: s_lshr_b32 s88, s21, 16
-; SI-NEXT: s_lshr_b32 s30, s20, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s20, 16
; SI-NEXT: s_lshr_b32 s79, s19, 16
; SI-NEXT: s_lshr_b32 s95, s18, 16
; SI-NEXT: s_lshr_b32 s78, s17, 16
; SI-NEXT: s_lshr_b32 s94, s16, 16
-; SI-NEXT: s_lshr_b32 s93, s37, 16
-; SI-NEXT: s_lshr_b32 s38, s39, 16
+; SI-NEXT: s_lshr_b32 s93, s35, 16
+; SI-NEXT: s_lshr_b32 s36, s37, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB89_4
+; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s78, 16
@@ -58055,40 +58985,40 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s7, s88, 16
; SI-NEXT: s_or_b32 s12, s4, s46
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s30, 16
+; SI-NEXT: s_lshl_b32 s56, vcc_lo, 16
; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s89, 16
; SI-NEXT: s_or_b32 s10, s4, s56
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s58, s31, 16
+; SI-NEXT: s_lshl_b32 s58, vcc_hi, 16
; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
; SI-NEXT: s_lshl_b32 s7, s90, 16
; SI-NEXT: s_or_b32 s8, s4, s58
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s34, 16
+; SI-NEXT: s_lshl_b32 s60, s30, 16
; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
; SI-NEXT: s_lshl_b32 s7, s91, 16
; SI-NEXT: s_or_b32 s6, s4, s60
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s35, 16
+; SI-NEXT: s_lshl_b32 s42, s31, 16
; SI-NEXT: s_or_b32 s43, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s92, 16
; SI-NEXT: s_or_b32 s4, s4, s42
-; SI-NEXT: s_lshl_b32 s40, s36, 16
+; SI-NEXT: s_lshl_b32 s40, s34, 16
; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s37, 0xffff
+; SI-NEXT: s_and_b32 s5, s35, 0xffff
; SI-NEXT: s_lshl_b32 s7, s93, 16
; SI-NEXT: s_lshr_b64 s[62:63], s[42:43], 16
; SI-NEXT: s_and_b32 s42, s28, 0xffff
; SI-NEXT: s_or_b32 s75, s5, s7
-; SI-NEXT: s_lshl_b32 s74, s38, 16
+; SI-NEXT: s_lshl_b32 s74, s36, 16
; SI-NEXT: s_or_b32 s42, s42, s40
; SI-NEXT: s_lshr_b64 s[72:73], s[40:41], 16
-; SI-NEXT: s_and_b32 s40, s39, 0xffff
+; SI-NEXT: s_and_b32 s40, s37, 0xffff
; SI-NEXT: s_mov_b32 s15, s45
; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 16
; SI-NEXT: s_mov_b32 s13, s47
@@ -58104,21 +59034,45 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s40, s40, s74
; SI-NEXT: s_mov_b32 s41, s75
; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
-; SI-NEXT: s_cbranch_execnz .LBB89_3
-; SI-NEXT: .LBB89_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s39, s39, 3
-; SI-NEXT: s_and_b32 s4, s39, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s38, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_mov_b64 s[76:77], 0
+; SI-NEXT: s_branch .LBB89_3
+; SI-NEXT: .LBB89_2:
+; SI-NEXT: s_mov_b64 s[76:77], -1
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: .LBB89_3: ; %Flow
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB89_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s37, s37, 3
-; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s40, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
@@ -58128,7 +59082,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s27, 0xffff
@@ -58136,7 +59090,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
+; SI-NEXT: s_lshl_b32 s7, s30, 16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s25, 0xffff
@@ -58144,7 +59098,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s31, 16
+; SI-NEXT: s_lshl_b32 s9, vcc_hi, 16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s23, 0xffff
@@ -58152,7 +59106,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s30, 16
+; SI-NEXT: s_lshl_b32 s11, vcc_lo, 16
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s21, 0xffff
@@ -58202,7 +59156,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s5, 16
; SI-NEXT: s_lshr_b32 s92, s43, 16
; SI-NEXT: s_lshr_b32 s93, s41, 16
-; SI-NEXT: .LBB89_3: ; %end
+; SI-NEXT: .LBB89_5: ; %end
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s16, s44, 16
; SI-NEXT: s_or_b32 s14, s14, s16
@@ -58251,7 +59205,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s19, s41, 0xffff
; SI-NEXT: s_lshl_b32 s20, s93, 16
; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: v_readlane_b32 s30, v16, 6
+; SI-NEXT: v_readlane_b32 s30, v16, 4
; SI-NEXT: v_mov_b32_e32 v0, s14
; SI-NEXT: v_mov_b32_e32 v1, s15
; SI-NEXT: v_mov_b32_e32 v2, s12
@@ -58268,9 +59222,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v13, s17
; SI-NEXT: v_mov_b32_e32 v14, s18
; SI-NEXT: v_mov_b32_e32 v15, s19
-; SI-NEXT: v_readlane_b32 s31, v16, 7
-; SI-NEXT: v_readlane_b32 s39, v16, 5
-; SI-NEXT: v_readlane_b32 s38, v16, 4
+; SI-NEXT: v_readlane_b32 s31, v16, 5
; SI-NEXT: v_readlane_b32 s37, v16, 3
; SI-NEXT: v_readlane_b32 s36, v16, 2
; SI-NEXT: v_readlane_b32 s35, v16, 1
@@ -58280,24 +59232,6 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB89_4:
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: s_branch .LBB89_2
;
; VI-LABEL: bitcast_v32i16_to_v32f16_scalar:
; VI: ; %bb.0:
@@ -58306,10 +59240,18 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB89_4
+; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB89_3
-; VI-NEXT: .LBB89_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB89_3
+; VI-NEXT: .LBB89_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB89_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB89_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s8, s17, 0xffff0000
@@ -58390,7 +59332,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s18, s10, 0x30000
; VI-NEXT: s_add_i32 s17, s8, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB89_3: ; %end
+; VI-NEXT: .LBB89_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -58408,8 +59350,6 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB89_4:
-; VI-NEXT: s_branch .LBB89_2
;
; GFX9-LABEL: bitcast_v32i16_to_v32f16_scalar:
; GFX9: ; %bb.0:
@@ -58423,10 +59363,18 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB89_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB89_4
-; GFX9-NEXT: .LBB89_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB89_3
+; GFX9-NEXT: .LBB89_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB89_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -58443,10 +59391,8 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB89_5
-; GFX9-NEXT: .LBB89_3:
-; GFX9-NEXT: s_branch .LBB89_2
-; GFX9-NEXT: .LBB89_4:
+; GFX9-NEXT: s_branch .LBB89_6
+; GFX9-NEXT: .LBB89_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -58463,7 +59409,7 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB89_5: ; %end
+; GFX9-NEXT: .LBB89_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -58481,11 +59427,16 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB89_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB89_4
-; GFX11-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB89_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -58503,8 +59454,6 @@ define inreg <32 x half> @bitcast_v32i16_to_v32f16_scalar(<32 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB89_3:
-; GFX11-NEXT: s_branch .LBB89_2
; GFX11-NEXT: .LBB89_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -58908,10 +59857,18 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s15, s6, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB91_3
+; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB91_4
-; SI-NEXT: .LBB91_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB91_3
+; SI-NEXT: .LBB91_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB91_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB91_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s46
; SI-NEXT: v_cvt_f32_f16_e32 v3, s45
@@ -59048,10 +60005,8 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[20:21], v[10:11], 16
; SI-NEXT: v_lshr_b64 v[18:19], v[12:13], 16
; SI-NEXT: v_lshr_b64 v[16:17], v[14:15], 16
-; SI-NEXT: s_branch .LBB91_5
-; SI-NEXT: .LBB91_3:
-; SI-NEXT: s_branch .LBB91_2
-; SI-NEXT: .LBB91_4:
+; SI-NEXT: s_branch .LBB91_6
+; SI-NEXT: .LBB91_5:
; SI-NEXT: v_mov_b32_e32 v32, s44
; SI-NEXT: v_mov_b32_e32 v38, s41
; SI-NEXT: v_mov_b32_e32 v33, s42
@@ -59084,7 +60039,7 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v20, s14
; SI-NEXT: v_mov_b32_e32 v18, s13
; SI-NEXT: v_mov_b32_e32 v16, s15
-; SI-NEXT: .LBB91_5: ; %end
+; SI-NEXT: .LBB91_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v30
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v55
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -59147,10 +60102,18 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB91_3
+; VI-NEXT: s_cbranch_scc0 .LBB91_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB91_4
-; VI-NEXT: .LBB91_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB91_3
+; VI-NEXT: .LBB91_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB91_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB91_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s30, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_mov_b32_e32 v3, s5
@@ -59232,10 +60195,8 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v1, s17, v1
; VI-NEXT: v_or_b32_e32 v1, v1, v17
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB91_5
-; VI-NEXT: .LBB91_3:
-; VI-NEXT: s_branch .LBB91_2
-; VI-NEXT: .LBB91_4:
+; VI-NEXT: s_branch .LBB91_6
+; VI-NEXT: .LBB91_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -59252,7 +60213,7 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB91_5: ; %end
+; VI-NEXT: .LBB91_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v18, 0
; VI-NEXT: v_readlane_b32 s31, v18, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -59273,10 +60234,18 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB91_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB91_4
-; GFX9-NEXT: .LBB91_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB91_3
+; GFX9-NEXT: .LBB91_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB91_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -59294,10 +60263,8 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB91_5
-; GFX9-NEXT: .LBB91_3:
-; GFX9-NEXT: s_branch .LBB91_2
-; GFX9-NEXT: .LBB91_4:
+; GFX9-NEXT: s_branch .LBB91_6
+; GFX9-NEXT: .LBB91_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -59314,7 +60281,7 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB91_5: ; %end
+; GFX9-NEXT: .LBB91_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -59332,11 +60299,16 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB91_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB91_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -59354,8 +60326,6 @@ define inreg <32 x i16> @bitcast_v32f16_to_v32i16_scalar(<32 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB91_3:
-; GFX11-NEXT: s_branch .LBB91_2
; GFX11-NEXT: .LBB91_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -59790,16 +60760,14 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: v_writelane_b32 v17, s35, 1
; SI-NEXT: v_writelane_b32 v17, s36, 2
; SI-NEXT: v_writelane_b32 v17, s37, 3
-; SI-NEXT: v_writelane_b32 v17, s38, 4
-; SI-NEXT: v_writelane_b32 v17, s39, 5
-; SI-NEXT: v_writelane_b32 v17, s30, 6
-; SI-NEXT: v_writelane_b32 v17, s31, 7
-; SI-NEXT: v_readfirstlane_b32 s39, v1
-; SI-NEXT: v_readfirstlane_b32 s37, v0
-; SI-NEXT: s_lshr_b32 s35, s29, 16
-; SI-NEXT: s_lshr_b32 s34, s28, 16
-; SI-NEXT: s_lshr_b32 s31, s27, 16
-; SI-NEXT: s_lshr_b32 s30, s26, 16
+; SI-NEXT: v_writelane_b32 v17, s30, 4
+; SI-NEXT: v_writelane_b32 v17, s31, 5
+; SI-NEXT: v_readfirstlane_b32 s37, v1
+; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: s_lshr_b32 s31, s29, 16
+; SI-NEXT: s_lshr_b32 s30, s28, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s26, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
; SI-NEXT: s_lshr_b32 s94, s24, 16
; SI-NEXT: s_lshr_b32 s93, s23, 16
@@ -59810,11 +60778,11 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: s_lshr_b32 s88, s18, 16
; SI-NEXT: s_lshr_b32 s79, s17, 16
; SI-NEXT: s_lshr_b32 s78, s16, 16
-; SI-NEXT: s_lshr_b32 s38, s39, 16
; SI-NEXT: s_lshr_b32 s36, s37, 16
+; SI-NEXT: s_lshr_b32 s34, s35, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB93_4
+; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s73, s16, 16
; SI-NEXT: s_lshl_b32 s77, s78, 16
@@ -59837,42 +60805,82 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: s_lshl_b32 s15, s25, 16
; SI-NEXT: s_lshl_b32 s46, s95, 16
; SI-NEXT: s_lshl_b32 s13, s26, 16
-; SI-NEXT: s_lshl_b32 s44, s30, 16
+; SI-NEXT: s_lshl_b32 s44, vcc_lo, 16
; SI-NEXT: s_lshl_b32 s11, s27, 16
-; SI-NEXT: s_lshl_b32 s42, s31, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_hi, 16
; SI-NEXT: s_lshl_b32 s10, s28, 16
-; SI-NEXT: s_lshl_b32 s40, s34, 16
+; SI-NEXT: s_lshl_b32 s40, s30, 16
; SI-NEXT: s_lshl_b32 s8, s29, 16
-; SI-NEXT: s_lshl_b32 s14, s35, 16
-; SI-NEXT: s_lshl_b32 s7, s37, 16
-; SI-NEXT: s_lshl_b32 s12, s36, 16
-; SI-NEXT: s_lshl_b32 s6, s39, 16
-; SI-NEXT: s_lshl_b32 s9, s38, 16
-; SI-NEXT: s_cbranch_execnz .LBB93_3
-; SI-NEXT: .LBB93_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s39, s39, 3
-; SI-NEXT: s_and_b32 s4, s39, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s38, 16
+; SI-NEXT: s_lshl_b32 s14, s31, 16
+; SI-NEXT: s_lshl_b32 s7, s35, 16
+; SI-NEXT: s_lshl_b32 s12, s34, 16
+; SI-NEXT: s_lshl_b32 s6, s37, 16
+; SI-NEXT: s_lshl_b32 s9, s36, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB93_3
+; SI-NEXT: .LBB93_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr43
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB93_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB93_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s37, s37, 3
+; SI-NEXT: s_and_b32 s4, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s37, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s36, 16
+; SI-NEXT: s_and_b32 s5, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s34, 16
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s35, 16
+; SI-NEXT: s_lshl_b32 s7, s31, 16
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s34, 16
+; SI-NEXT: s_lshl_b32 s8, s30, 16
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s31, 16
+; SI-NEXT: s_lshl_b32 s9, vcc_hi, 16
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s30, 16
+; SI-NEXT: s_lshl_b32 s10, vcc_lo, 16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s25, 0xffff
@@ -59962,7 +60970,7 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s4, 16
-; SI-NEXT: .LBB93_3: ; %end
+; SI-NEXT: .LBB93_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
@@ -60026,11 +61034,9 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s6
-; SI-NEXT: v_readlane_b32 s30, v17, 6
+; SI-NEXT: v_readlane_b32 s30, v17, 4
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_readlane_b32 s31, v17, 7
-; SI-NEXT: v_readlane_b32 s39, v17, 5
-; SI-NEXT: v_readlane_b32 s38, v17, 4
+; SI-NEXT: v_readlane_b32 s31, v17, 5
; SI-NEXT: v_readlane_b32 s37, v17, 3
; SI-NEXT: v_readlane_b32 s36, v17, 2
; SI-NEXT: v_readlane_b32 s35, v17, 1
@@ -60040,40 +61046,6 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB93_4:
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB93_2
;
; VI-LABEL: bitcast_v32i16_to_v32bf16_scalar:
; VI: ; %bb.0:
@@ -60082,10 +61054,18 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; VI-NEXT: v_readfirstlane_b32 s6, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: s_cbranch_scc0 .LBB93_4
+; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB93_3
-; VI-NEXT: .LBB93_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB93_3
+; VI-NEXT: .LBB93_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB93_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB93_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_and_b32 s8, s17, 0xffff0000
@@ -60166,7 +61146,7 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; VI-NEXT: s_add_i32 s18, s10, 0x30000
; VI-NEXT: s_add_i32 s17, s8, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB93_3: ; %end
+; VI-NEXT: .LBB93_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -60184,8 +61164,6 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_mov_b32_e32 v15, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB93_4:
-; VI-NEXT: s_branch .LBB93_2
;
; GFX9-LABEL: bitcast_v32i16_to_v32bf16_scalar:
; GFX9: ; %bb.0:
@@ -60199,10 +61177,18 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB93_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB93_4
-; GFX9-NEXT: .LBB93_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB93_3
+; GFX9-NEXT: .LBB93_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB93_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v15, s31, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s30, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s29, 3 op_sel_hi:[1,0]
@@ -60219,10 +61205,8 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB93_5
-; GFX9-NEXT: .LBB93_3:
-; GFX9-NEXT: s_branch .LBB93_2
-; GFX9-NEXT: .LBB93_4:
+; GFX9-NEXT: s_branch .LBB93_6
+; GFX9-NEXT: .LBB93_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -60239,7 +61223,7 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB93_5: ; %end
+; GFX9-NEXT: .LBB93_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -60257,11 +61241,16 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB93_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB93_4
-; GFX11-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB93_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v15, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s25, 3 op_sel_hi:[1,0]
@@ -60279,8 +61268,6 @@ define inreg <32 x bfloat> @bitcast_v32i16_to_v32bf16_scalar(<32 x i16> inreg %a
; GFX11-NEXT: v_pk_add_u16 v1, s13, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s12, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB93_3:
-; GFX11-NEXT: s_branch .LBB93_2
; GFX11-NEXT: .LBB93_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -61871,7 +62858,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v28, 1.0, s45
; SI-NEXT: v_mul_f32_e64 v55, 1.0, s42
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
-; SI-NEXT: s_cbranch_scc0 .LBB95_4
+; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v63
; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v6
@@ -61887,6 +62874,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v8
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v58
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v57
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v20
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v56
; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v47
@@ -61905,8 +62893,47 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v32
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v30
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v55
-; SI-NEXT: s_cbranch_execnz .LBB95_3
-; SI-NEXT: .LBB95_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB95_3
+; SI-NEXT: .LBB95_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr52
+; SI-NEXT: ; implicit-def: $vgpr51
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr54
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr23
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr24
+; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr43
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr19
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr15
+; SI-NEXT: .LBB95_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB95_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v63
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
@@ -62019,7 +63046,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_lshr_b64 v[42:43], v[16:17], 16
; SI-NEXT: v_lshr_b64 v[54:55], v[20:21], 16
; SI-NEXT: v_lshr_b64 v[43:44], v[14:15], 16
-; SI-NEXT: .LBB95_3: ; %end
+; SI-NEXT: .LBB95_5: ; %end
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
@@ -62086,40 +63113,6 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB95_4:
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr52
-; SI-NEXT: ; implicit-def: $vgpr51
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $vgpr23
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $vgpr41
-; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr31
-; SI-NEXT: ; implicit-def: $vgpr27
-; SI-NEXT: ; implicit-def: $vgpr43
-; SI-NEXT: ; implicit-def: $vgpr29
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr19
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: s_branch .LBB95_2
;
; VI-LABEL: bitcast_v32bf16_to_v32i16_scalar:
; VI: ; %bb.0:
@@ -62133,10 +63126,18 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB95_3
+; VI-NEXT: s_cbranch_scc0 .LBB95_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB95_4
-; VI-NEXT: .LBB95_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB95_3
+; VI-NEXT: .LBB95_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB95_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB95_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s26, 16
; VI-NEXT: v_add_f32_e32 v4, s4, v16
@@ -62433,10 +63434,8 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_lshrrev_b64 v[17:18], 16, v[17:18]
; VI-NEXT: v_lshrrev_b64 v[15:16], 16, v[15:16]
; VI-NEXT: v_mov_b32_e32 v13, v17
-; VI-NEXT: s_branch .LBB95_5
-; VI-NEXT: .LBB95_3:
-; VI-NEXT: s_branch .LBB95_2
-; VI-NEXT: .LBB95_4:
+; VI-NEXT: s_branch .LBB95_6
+; VI-NEXT: .LBB95_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -62453,7 +63452,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB95_5: ; %end
+; VI-NEXT: .LBB95_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -62474,10 +63473,18 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB95_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB95_4
-; GFX9-NEXT: .LBB95_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB95_3
+; GFX9-NEXT: .LBB95_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB95_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_add_f32_e32 v1, s5, v0
@@ -62768,10 +63775,8 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_and_or_b32 v0, v17, v16, v0
-; GFX9-NEXT: s_branch .LBB95_5
-; GFX9-NEXT: .LBB95_3:
-; GFX9-NEXT: s_branch .LBB95_2
-; GFX9-NEXT: .LBB95_4:
+; GFX9-NEXT: s_branch .LBB95_6
+; GFX9-NEXT: .LBB95_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -62788,7 +63793,7 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB95_5: ; %end
+; GFX9-NEXT: .LBB95_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -62806,11 +63811,16 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-TRUE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s12, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s12, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -63079,8 +64089,6 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v32, v37, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB95_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB95_2
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -63101,11 +64109,16 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-FAKE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s0, s12, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s12, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -63399,8 +64412,6 @@ define inreg <32 x i16> @bitcast_v32bf16_to_v32i16_scalar(<32 x bfloat> inreg %a
; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v16, v23
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v0, v24
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB95_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB95_2
; GFX11-FAKE16-NEXT: .LBB95_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -65458,7 +66469,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: ; implicit-def: $vgpr5 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB97_4
+; SI-NEXT: s_cbranch_scc0 .LBB97_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s84, 16
@@ -65557,8 +66568,75 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 24
; SI-NEXT: s_lshr_b64 s[34:35], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[36:37], s[4:5], 8
-; SI-NEXT: s_cbranch_execnz .LBB97_3
-; SI-NEXT: .LBB97_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 vcc, 0
+; SI-NEXT: s_branch .LBB97_3
+; SI-NEXT: .LBB97_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b64 vcc, -1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v5, s4, 0
+; SI-NEXT: v_writelane_b32 v5, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: v_writelane_b32 v5, s4, 2
+; SI-NEXT: v_writelane_b32 v5, s5, 3
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v5, s4, 4
+; SI-NEXT: v_writelane_b32 v5, s5, 5
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v5, s4, 6
+; SI-NEXT: v_writelane_b32 v5, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB97_3: ; %Flow
+; SI-NEXT: s_and_b64 vcc, vcc, exec
+; SI-NEXT: s_cselect_b32 vcc_lo, 1, 0
+; SI-NEXT: s_cmp_lg_u32 vcc_lo, 1
+; SI-NEXT: v_readlane_b32 vcc_lo, v5, 2
+; SI-NEXT: v_readlane_b32 vcc_hi, v5, 3
+; SI-NEXT: s_cbranch_scc1 .LBB97_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s58, s58, 3
; SI-NEXT: s_and_b32 s4, s58, 0xffff
; SI-NEXT: s_lshl_b32 s5, s56, 16
@@ -65631,9 +66709,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s17, s83, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s43, s16, 0x30000
-; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 24
-; SI-NEXT: v_writelane_b32 v5, s16, 2
-; SI-NEXT: v_writelane_b32 v5, s17, 3
; SI-NEXT: s_lshr_b64 s[16:17], s[42:43], 16
; SI-NEXT: v_writelane_b32 v5, s16, 0
; SI-NEXT: v_writelane_b32 v5, s17, 1
@@ -65659,6 +66734,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[94:95], s[8:9], 8
; SI-NEXT: s_lshr_b64 s[88:89], s[14:15], 24
; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
+; SI-NEXT: s_lshr_b64 vcc, s[42:43], 24
; SI-NEXT: v_writelane_b32 v5, s16, 6
; SI-NEXT: s_lshr_b64 s[66:67], s[14:15], 8
; SI-NEXT: s_lshr_b64 s[52:53], s[10:11], 24
@@ -65695,16 +66771,14 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s69, s5, 16
; SI-NEXT: s_lshr_b32 s45, s5, 8
; SI-NEXT: v_writelane_b32 v5, s17, 7
-; SI-NEXT: .LBB97_3: ; %end
-; SI-NEXT: v_readlane_b32 s18, v5, 0
+; SI-NEXT: .LBB97_5: ; %end
; SI-NEXT: s_and_b32 s16, s42, 0xff
; SI-NEXT: s_lshl_b32 s17, s62, 8
-; SI-NEXT: v_readlane_b32 s19, v5, 1
+; SI-NEXT: v_readlane_b32 s18, v5, 0
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s18, 0xff
-; SI-NEXT: v_readlane_b32 s18, v5, 2
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s18, 24
+; SI-NEXT: s_lshl_b32 s18, vcc_lo, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -65718,7 +66792,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s19, v5, 3
+; SI-NEXT: v_readlane_b32 s19, v5, 1
; SI-NEXT: v_mov_b32_e32 v2, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
; SI-NEXT: s_lshl_b32 s17, s74, 8
@@ -65951,65 +67025,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB97_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v5, s4, 0
-; SI-NEXT: v_writelane_b32 v5, s5, 1
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr99
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: v_writelane_b32 v5, s4, 2
-; SI-NEXT: v_writelane_b32 v5, s5, 3
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v5, s4, 4
-; SI-NEXT: v_writelane_b32 v5, s5, 5
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v5, s4, 6
-; SI-NEXT: v_writelane_b32 v5, s5, 7
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB97_2
;
; VI-LABEL: bitcast_v32i16_to_v64i8_scalar:
; VI: ; %bb.0:
@@ -66033,15 +67048,13 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: v_writelane_b32 v4, s55, 13
; VI-NEXT: v_writelane_b32 v4, s64, 14
; VI-NEXT: v_writelane_b32 v4, s65, 15
-; VI-NEXT: v_writelane_b32 v4, s66, 16
-; VI-NEXT: v_writelane_b32 v4, s67, 17
-; VI-NEXT: v_writelane_b32 v4, s30, 18
-; VI-NEXT: v_writelane_b32 v4, s31, 19
+; VI-NEXT: v_writelane_b32 v4, s30, 16
+; VI-NEXT: v_writelane_b32 v4, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB97_4
+; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s56, s5, 24
; VI-NEXT: s_lshr_b32 s57, s5, 16
@@ -66078,11 +67091,11 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[28:29], 24
; VI-NEXT: s_lshr_b64 s[10:11], s[26:27], 24
@@ -66091,8 +67104,64 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[40:41], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[44:45], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB97_3
-; VI-NEXT: .LBB97_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB97_3
+; VI-NEXT: .LBB97_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: .LBB97_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB97_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_and_b32 s6, s17, 0xffff0000
; VI-NEXT: s_and_b32 s7, s7, 0xffff
@@ -66216,23 +67285,23 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b32 s52, s19, 8
; VI-NEXT: s_lshr_b32 s53, s18, 16
; VI-NEXT: s_lshr_b32 s54, s18, 8
-; VI-NEXT: s_lshr_b32 s55, s17, 24
-; VI-NEXT: s_lshr_b32 s64, s17, 16
-; VI-NEXT: s_lshr_b32 s65, s17, 8
-; VI-NEXT: s_lshr_b32 s66, s16, 16
-; VI-NEXT: s_lshr_b32 s67, s16, 8
-; VI-NEXT: .LBB97_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s17, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s17, 16
+; VI-NEXT: s_lshr_b32 s55, s17, 8
+; VI-NEXT: s_lshr_b32 s64, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 8
+; VI-NEXT: .LBB97_5: ; %end
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v3, s44
-; VI-NEXT: v_mov_b32_e32 v1, s67
-; VI-NEXT: v_perm_b32 v3, s66, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s65
+; VI-NEXT: v_perm_b32 v3, s64, v3, v2
; VI-NEXT: v_perm_b32 v1, s16, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
-; VI-NEXT: v_mov_b32_e32 v3, s55
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_mov_b32_e32 v1, s65
-; VI-NEXT: v_perm_b32 v3, s64, v3, v2
+; VI-NEXT: v_mov_b32_e32 v1, s55
+; VI-NEXT: v_perm_b32 v3, vcc_hi, v3, v2
; VI-NEXT: v_perm_b32 v1, s17, v1, v2
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -66349,11 +67418,9 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: v_add_u32_e32 v0, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s30, v4, 18
+; VI-NEXT: v_readlane_b32 s30, v4, 16
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
-; VI-NEXT: v_readlane_b32 s31, v4, 19
-; VI-NEXT: v_readlane_b32 s67, v4, 17
-; VI-NEXT: v_readlane_b32 s66, v4, 16
+; VI-NEXT: v_readlane_b32 s31, v4, 17
; VI-NEXT: v_readlane_b32 s65, v4, 15
; VI-NEXT: v_readlane_b32 s64, v4, 14
; VI-NEXT: v_readlane_b32 s55, v4, 13
@@ -66375,56 +67442,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB97_4:
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: s_branch .LBB97_2
;
; GFX9-LABEL: bitcast_v32i16_to_v64i8_scalar:
; GFX9: ; %bb.0:
@@ -66459,15 +67476,13 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_writelane_b32 v63, s51, 9
; GFX9-NEXT: v_writelane_b32 v63, s52, 10
; GFX9-NEXT: v_writelane_b32 v63, s53, 11
-; GFX9-NEXT: v_writelane_b32 v63, s54, 12
-; GFX9-NEXT: v_writelane_b32 v63, s55, 13
-; GFX9-NEXT: v_writelane_b32 v63, s30, 14
-; GFX9-NEXT: v_writelane_b32 v63, s31, 15
+; GFX9-NEXT: v_writelane_b32 v63, s30, 12
+; GFX9-NEXT: v_writelane_b32 v63, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s56, s5, 24
; GFX9-NEXT: s_lshr_b32 s57, s5, 16
@@ -66493,22 +67508,22 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s95, s23, 8
; GFX9-NEXT: s_lshr_b32 s94, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s36, s21, 8
-; GFX9-NEXT: s_lshr_b32 s35, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s49, s19, 8
-; GFX9-NEXT: s_lshr_b32 s48, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s54, s17, 8
-; GFX9-NEXT: s_lshr_b32 s53, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s34, s21, 8
+; GFX9-NEXT: s_lshr_b32 s31, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s39, s19, 8
+; GFX9-NEXT: s_lshr_b32 s38, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s52, s17, 8
+; GFX9-NEXT: s_lshr_b32 s51, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
; GFX9-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -66517,8 +67532,64 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB97_4
-; GFX9-NEXT: .LBB97_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB97_3
+; GFX9-NEXT: .LBB97_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: .LBB97_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v6, s27, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v5, s26, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s5, 3 op_sel_hi:[1,0]
@@ -66589,58 +67660,8 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v18
-; GFX9-NEXT: s_branch .LBB97_5
-; GFX9-NEXT: .LBB97_3:
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: s_branch .LBB97_2
-; GFX9-NEXT: .LBB97_4:
+; GFX9-NEXT: s_branch .LBB97_6
+; GFX9-NEXT: .LBB97_5:
; GFX9-NEXT: v_mov_b32_e32 v20, s44
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -66662,22 +67683,22 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s29
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NEXT: v_mov_b32_e32 v15, s55
-; GFX9-NEXT: v_mov_b32_e32 v16, s53
-; GFX9-NEXT: v_mov_b32_e32 v62, s54
-; GFX9-NEXT: v_mov_b32_e32 v59, s52
-; GFX9-NEXT: v_mov_b32_e32 v60, s51
-; GFX9-NEXT: v_mov_b32_e32 v58, s50
-; GFX9-NEXT: v_mov_b32_e32 v61, s48
-; GFX9-NEXT: v_mov_b32_e32 v57, s49
-; GFX9-NEXT: v_mov_b32_e32 v47, s39
-; GFX9-NEXT: v_mov_b32_e32 v56, s38
-; GFX9-NEXT: v_mov_b32_e32 v46, s37
-; GFX9-NEXT: v_mov_b32_e32 v45, s35
-; GFX9-NEXT: v_mov_b32_e32 v44, s36
-; GFX9-NEXT: v_mov_b32_e32 v42, s34
-; GFX9-NEXT: v_mov_b32_e32 v43, s31
-; GFX9-NEXT: v_mov_b32_e32 v41, s30
+; GFX9-NEXT: v_mov_b32_e32 v15, s53
+; GFX9-NEXT: v_mov_b32_e32 v16, s51
+; GFX9-NEXT: v_mov_b32_e32 v62, s52
+; GFX9-NEXT: v_mov_b32_e32 v59, s50
+; GFX9-NEXT: v_mov_b32_e32 v60, s49
+; GFX9-NEXT: v_mov_b32_e32 v58, s48
+; GFX9-NEXT: v_mov_b32_e32 v61, s38
+; GFX9-NEXT: v_mov_b32_e32 v57, s39
+; GFX9-NEXT: v_mov_b32_e32 v47, s37
+; GFX9-NEXT: v_mov_b32_e32 v56, s36
+; GFX9-NEXT: v_mov_b32_e32 v46, s35
+; GFX9-NEXT: v_mov_b32_e32 v45, s31
+; GFX9-NEXT: v_mov_b32_e32 v44, s34
+; GFX9-NEXT: v_mov_b32_e32 v42, s30
+; GFX9-NEXT: v_mov_b32_e32 v43, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v41, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v40, s94
; GFX9-NEXT: v_mov_b32_e32 v55, s95
; GFX9-NEXT: v_mov_b32_e32 v53, s93
@@ -66711,7 +67732,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v20, s40
; GFX9-NEXT: v_mov_b32_e32 v21, s14
-; GFX9-NEXT: .LBB97_5: ; %end
+; GFX9-NEXT: .LBB97_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v16, v25, s4
; GFX9-NEXT: v_perm_b32 v15, v18, v15, s4
@@ -66778,10 +67799,8 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
-; GFX9-NEXT: v_readlane_b32 s30, v63, 14
-; GFX9-NEXT: v_readlane_b32 s31, v63, 15
-; GFX9-NEXT: v_readlane_b32 s55, v63, 13
-; GFX9-NEXT: v_readlane_b32 s54, v63, 12
+; GFX9-NEXT: v_readlane_b32 s30, v63, 12
+; GFX9-NEXT: v_readlane_b32 s31, v63, 13
; GFX9-NEXT: v_readlane_b32 s53, v63, 11
; GFX9-NEXT: v_readlane_b32 s52, v63, 10
; GFX9-NEXT: v_readlane_b32 s51, v63, 9
@@ -66851,12 +67870,11 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s43, s27, 24
; GFX11-NEXT: s_lshr_b32 s44, s27, 16
@@ -66887,17 +67905,17 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s93, s17, 16
; GFX11-NEXT: s_lshr_b32 s95, s17, 8
; GFX11-NEXT: s_lshr_b32 s94, s16, 16
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s16, 8
-; GFX11-NEXT: s_lshr_b32 s30, s3, 24
-; GFX11-NEXT: s_lshr_b32 s31, s3, 16
-; GFX11-NEXT: s_lshr_b32 s35, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s36, s2, 8
-; GFX11-NEXT: s_lshr_b32 s37, s1, 24
-; GFX11-NEXT: s_lshr_b32 s38, s1, 16
-; GFX11-NEXT: s_lshr_b32 s48, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s49, s0, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s16, 8
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
+; GFX11-NEXT: s_lshr_b32 s30, s3, 16
+; GFX11-NEXT: s_lshr_b32 s34, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s35, s2, 8
+; GFX11-NEXT: s_lshr_b32 s36, s1, 24
+; GFX11-NEXT: s_lshr_b32 s37, s1, 16
+; GFX11-NEXT: s_lshr_b32 s39, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s48, s0, 8
; GFX11-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[14:15], s[22:23], 24
@@ -66906,9 +67924,64 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB97_4
-; GFX11-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB97_3
+; GFX11-NEXT: .LBB97_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr95
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: .LBB97_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v4, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v3, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v6, s23, 3 op_sel_hi:[1,0]
@@ -66973,58 +68046,8 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v83, 8, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v96, 16, v17
; GFX11-NEXT: v_lshrrev_b32_e32 v85, 8, v17
-; GFX11-NEXT: s_branch .LBB97_5
-; GFX11-NEXT: .LBB97_3:
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr31
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr95
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: s_branch .LBB97_2
-; GFX11-NEXT: .LBB97_4:
+; GFX11-NEXT: s_branch .LBB97_6
+; GFX11-NEXT: .LBB97_5:
; GFX11-NEXT: v_dual_mov_b32 v17, s0 :: v_dual_mov_b32 v18, s1
; GFX11-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s3
; GFX11-NEXT: v_dual_mov_b32 v11, s16 :: v_dual_mov_b32 v12, s17
@@ -67033,12 +68056,12 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v5, s22 :: v_dual_mov_b32 v6, s23
; GFX11-NEXT: v_dual_mov_b32 v3, s24 :: v_dual_mov_b32 v4, s25
; GFX11-NEXT: v_dual_mov_b32 v1, s26 :: v_dual_mov_b32 v2, s27
-; GFX11-NEXT: v_dual_mov_b32 v85, s49 :: v_dual_mov_b32 v96, s39
-; GFX11-NEXT: v_dual_mov_b32 v83, s48 :: v_dual_mov_b32 v86, s38
-; GFX11-NEXT: v_dual_mov_b32 v87, s37 :: v_dual_mov_b32 v82, s36
-; GFX11-NEXT: v_dual_mov_b32 v84, s34 :: v_dual_mov_b32 v71, s35
-; GFX11-NEXT: v_dual_mov_b32 v80, s31 :: v_dual_mov_b32 v81, s30
-; GFX11-NEXT: v_dual_mov_b32 v67, vcc_hi :: v_dual_mov_b32 v70, s94
+; GFX11-NEXT: v_dual_mov_b32 v85, s48 :: v_dual_mov_b32 v96, s38
+; GFX11-NEXT: v_dual_mov_b32 v83, s39 :: v_dual_mov_b32 v86, s37
+; GFX11-NEXT: v_dual_mov_b32 v87, s36 :: v_dual_mov_b32 v82, s35
+; GFX11-NEXT: v_dual_mov_b32 v84, s31 :: v_dual_mov_b32 v71, s34
+; GFX11-NEXT: v_dual_mov_b32 v80, s30 :: v_dual_mov_b32 v81, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v67, vcc_lo :: v_dual_mov_b32 v70, s94
; GFX11-NEXT: v_dual_mov_b32 v65, s95 :: v_dual_mov_b32 v68, s93
; GFX11-NEXT: v_dual_mov_b32 v69, s92 :: v_dual_mov_b32 v64, s91
; GFX11-NEXT: v_dual_mov_b32 v66, s89 :: v_dual_mov_b32 v53, s90
@@ -67057,7 +68080,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v20, s14 :: v_dual_mov_b32 v21, s12
; GFX11-NEXT: v_dual_mov_b32 v22, s10 :: v_dual_mov_b32 v23, s8
; GFX11-NEXT: v_dual_mov_b32 v24, s6 :: v_dual_mov_b32 v25, s4
-; GFX11-NEXT: .LBB97_5: ; %end
+; GFX11-NEXT: .LBB97_6: ; %end
; GFX11-NEXT: v_perm_b32 v86, v86, v87, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_perm_b32 v24, v84, v24, 0xc0c0004
@@ -67123,14 +68146,13 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v2, v4, v17
; GFX11-NEXT: v_or_b32_e32 v3, v19, v15
; GFX11-NEXT: v_or_b32_e32 v4, v16, v18
-; GFX11-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: scratch_store_b128 v0, v[80:83], off
; GFX11-NEXT: scratch_store_b128 v0, v[11:14], off offset:16
; GFX11-NEXT: scratch_store_b128 v0, v[7:10], off offset:32
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:48
-; GFX11-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-NEXT: v_readlane_b32 s38, v40, 4
@@ -69711,276 +70733,274 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr44 : SGPR spill to VGPR lane
-; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v44, s23, 0
-; SI-NEXT: v_writelane_b32 v44, s22, 1
-; SI-NEXT: v_writelane_b32 v44, s21, 2
-; SI-NEXT: v_writelane_b32 v44, s20, 3
-; SI-NEXT: v_writelane_b32 v44, s4, 4
-; SI-NEXT: s_mov_b32 s82, s29
-; SI-NEXT: s_mov_b32 s85, s25
-; SI-NEXT: s_mov_b32 s29, s24
-; SI-NEXT: s_mov_b32 s44, s16
-; SI-NEXT: v_readfirstlane_b32 s64, v30
+; SI-NEXT: s_mov_b32 s96, s27
+; SI-NEXT: v_writelane_b32 v44, s28, 0
+; SI-NEXT: v_writelane_b32 v44, s26, 1
+; SI-NEXT: v_writelane_b32 v44, s23, 2
+; SI-NEXT: v_writelane_b32 v44, s22, 3
+; SI-NEXT: v_writelane_b32 v44, s16, 4
+; SI-NEXT: s_mov_b32 s27, s25
+; SI-NEXT: s_mov_b32 s85, s24
+; SI-NEXT: v_readfirstlane_b32 s69, v30
; SI-NEXT: v_readfirstlane_b32 s55, v29
-; SI-NEXT: v_readfirstlane_b32 s66, v28
-; SI-NEXT: v_readfirstlane_b32 s69, v27
-; SI-NEXT: v_readfirstlane_b32 s68, v26
-; SI-NEXT: v_readfirstlane_b32 s39, v25
-; SI-NEXT: v_readfirstlane_b32 s50, v24
-; SI-NEXT: v_readfirstlane_b32 s65, v23
-; SI-NEXT: v_readfirstlane_b32 s51, v22
-; SI-NEXT: v_readfirstlane_b32 s37, v21
-; SI-NEXT: v_readfirstlane_b32 s53, v20
-; SI-NEXT: v_readfirstlane_b32 s88, v19
-; SI-NEXT: v_readfirstlane_b32 s31, v18
-; SI-NEXT: v_readfirstlane_b32 s95, v17
-; SI-NEXT: v_readfirstlane_b32 s48, v15
-; SI-NEXT: v_readfirstlane_b32 s35, v14
+; SI-NEXT: v_readfirstlane_b32 s65, v28
+; SI-NEXT: v_readfirstlane_b32 s71, v27
+; SI-NEXT: v_readfirstlane_b32 s67, v26
+; SI-NEXT: v_readfirstlane_b32 s38, v25
+; SI-NEXT: v_readfirstlane_b32 s68, v24
+; SI-NEXT: v_readfirstlane_b32 s49, v23
+; SI-NEXT: v_readfirstlane_b32 s94, v22
+; SI-NEXT: v_readfirstlane_b32 s50, v21
+; SI-NEXT: v_readfirstlane_b32 s88, v20
+; SI-NEXT: v_readfirstlane_b32 s66, v19
+; SI-NEXT: v_readfirstlane_b32 s54, v18
+; SI-NEXT: v_readfirstlane_b32 s86, v17
+; SI-NEXT: v_readfirstlane_b32 s31, v16
+; SI-NEXT: v_readfirstlane_b32 s37, v15
+; SI-NEXT: v_readfirstlane_b32 s39, v14
; SI-NEXT: v_readfirstlane_b32 s34, v13
-; SI-NEXT: v_readfirstlane_b32 s52, v12
-; SI-NEXT: v_readfirstlane_b32 s16, v11
-; SI-NEXT: v_readfirstlane_b32 s96, v10
-; SI-NEXT: v_readfirstlane_b32 s20, v9
-; SI-NEXT: v_readfirstlane_b32 s89, v8
-; SI-NEXT: v_readfirstlane_b32 s92, v7
+; SI-NEXT: v_readfirstlane_b32 s36, v12
+; SI-NEXT: v_readfirstlane_b32 s53, v11
+; SI-NEXT: v_readfirstlane_b32 s64, v10
+; SI-NEXT: v_readfirstlane_b32 s89, v9
+; SI-NEXT: v_readfirstlane_b32 s90, v8
+; SI-NEXT: v_readfirstlane_b32 s16, v7
; SI-NEXT: v_readfirstlane_b32 s91, v6
-; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s38, v3
-; SI-NEXT: v_readfirstlane_b32 s49, v2
-; SI-NEXT: v_readfirstlane_b32 s54, v1
-; SI-NEXT: v_readfirstlane_b32 s90, v0
+; SI-NEXT: v_readfirstlane_b32 s28, v5
+; SI-NEXT: v_readfirstlane_b32 s93, v4
+; SI-NEXT: v_readfirstlane_b32 s35, v3
+; SI-NEXT: v_readfirstlane_b32 s51, v2
+; SI-NEXT: v_readfirstlane_b32 s23, v1
+; SI-NEXT: v_readfirstlane_b32 s79, v0
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: v_readfirstlane_b32 s58, v32
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 s56, v32
-; SI-NEXT: v_readfirstlane_b32 s86, v33
-; SI-NEXT: v_readfirstlane_b32 s58, v34
-; SI-NEXT: v_readfirstlane_b32 s99, v35
-; SI-NEXT: v_readfirstlane_b32 s24, v36
+; SI-NEXT: v_readfirstlane_b32 s84, v33
+; SI-NEXT: v_readfirstlane_b32 s47, v34
+; SI-NEXT: v_readfirstlane_b32 s30, v35
+; SI-NEXT: v_readfirstlane_b32 s25, v36
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_readfirstlane_b32 s78, v37
+; SI-NEXT: v_readfirstlane_b32 s98, v37
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s25, v38
+; SI-NEXT: v_readfirstlane_b32 s26, v38
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s21, v39
+; SI-NEXT: v_readfirstlane_b32 s22, v39
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s22, v48
+; SI-NEXT: v_readfirstlane_b32 s99, v48
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s23, v49
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s87, v50
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s47, v52
-; SI-NEXT: v_readfirstlane_b32 s83, v51
+; SI-NEXT: v_readfirstlane_b32 s70, v49
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_readfirstlane_b32 s56, v51
+; SI-NEXT: v_writelane_b32 v44, s56, 5
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s84, v53
+; SI-NEXT: v_readfirstlane_b32 s57, v53
+; SI-NEXT: v_writelane_b32 v44, s57, 6
+; SI-NEXT: v_readfirstlane_b32 s59, v52
+; SI-NEXT: v_writelane_b32 v44, s58, 7
+; SI-NEXT: v_readfirstlane_b32 s78, v50
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s36, v54
+; SI-NEXT: v_readfirstlane_b32 s44, v54
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s79, v55
+; SI-NEXT: v_readfirstlane_b32 s87, v55
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s71, v40
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v40
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s98, v41
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v41
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s81, v42
-; SI-NEXT: v_writelane_b32 v44, s47, 5
-; SI-NEXT: v_writelane_b32 v44, s58, 6
+; SI-NEXT: v_readfirstlane_b32 s83, v42
+; SI-NEXT: v_writelane_b32 v44, s59, 8
+; SI-NEXT: v_writelane_b32 v44, s47, 9
; SI-NEXT: s_cbranch_scc0 .LBB99_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s44, 0xff
+; SI-NEXT: v_readlane_b32 s4, v44, 4
+; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_or_b32 s12, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s19, 24
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s29, 0xff
-; SI-NEXT: s_lshl_b32 s6, s85, 8
+; SI-NEXT: s_and_b32 s5, s85, 0xff
+; SI-NEXT: s_lshl_b32 s6, s27, 8
; SI-NEXT: s_or_b32 s13, s5, s6
-; SI-NEXT: s_and_b32 s5, s26, 0xff
+; SI-NEXT: v_readlane_b32 s5, v44, 1
+; SI-NEXT: s_and_b32 s5, s5, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s27, 24
+; SI-NEXT: s_lshl_b32 s6, s96, 24
; SI-NEXT: s_or_b32 s6, s6, s5
-; SI-NEXT: s_and_b32 s5, s49, 0xff
-; SI-NEXT: s_lshl_b32 s7, s38, 8
+; SI-NEXT: s_and_b32 s5, s51, 0xff
+; SI-NEXT: s_lshl_b32 s7, s35, 8
; SI-NEXT: s_or_b32 s14, s5, s7
-; SI-NEXT: s_and_b32 s5, s30, 0xff
+; SI-NEXT: s_and_b32 s5, s93, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s93, 24
+; SI-NEXT: s_lshl_b32 s7, s28, 24
; SI-NEXT: s_or_b32 s8, s7, s5
-; SI-NEXT: s_and_b32 s5, s96, 0xff
-; SI-NEXT: s_lshl_b32 s7, s16, 8
+; SI-NEXT: s_and_b32 s5, s64, 0xff
+; SI-NEXT: s_lshl_b32 s7, s53, 8
; SI-NEXT: s_or_b32 s15, s5, s7
-; SI-NEXT: s_and_b32 s5, s52, 0xff
+; SI-NEXT: s_and_b32 s5, s36, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s34, 24
; SI-NEXT: s_or_b32 s10, s7, s5
-; SI-NEXT: s_and_b32 s5, s31, 0xff
-; SI-NEXT: s_lshl_b32 s7, s88, 8
+; SI-NEXT: s_and_b32 s5, s54, 0xff
+; SI-NEXT: s_lshl_b32 s7, s66, 8
; SI-NEXT: s_or_b32 s40, s5, s7
-; SI-NEXT: s_and_b32 s5, s53, 0xff
+; SI-NEXT: s_and_b32 s5, s88, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s37, 24
+; SI-NEXT: s_lshl_b32 s7, s50, 24
; SI-NEXT: s_or_b32 s60, s7, s5
-; SI-NEXT: s_and_b32 s5, s68, 0xff
-; SI-NEXT: s_lshl_b32 s7, s69, 8
+; SI-NEXT: s_and_b32 s5, s67, 0xff
+; SI-NEXT: s_lshl_b32 s7, s71, 8
; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s66, 0xff
+; SI-NEXT: s_and_b32 s5, s65, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s55, 24
; SI-NEXT: s_or_b32 s62, s7, s5
-; SI-NEXT: s_and_b32 s5, s79, 0xff
-; SI-NEXT: s_lshl_b32 s7, s36, 8
+; SI-NEXT: s_and_b32 s5, s87, 0xff
+; SI-NEXT: s_lshl_b32 s7, s44, 8
; SI-NEXT: s_or_b32 s42, s5, s7
-; SI-NEXT: s_and_b32 s5, s21, 0xff
-; SI-NEXT: s_lshl_b32 s7, s25, 8
+; SI-NEXT: s_and_b32 s5, s22, 0xff
+; SI-NEXT: s_lshl_b32 s7, s26, 8
; SI-NEXT: s_or_b32 s43, s5, s7
-; SI-NEXT: v_readlane_b32 s5, v44, 3
-; SI-NEXT: v_readlane_b32 s7, v44, 2
-; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s7, s21, 8
; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readlane_b32 s7, v44, 1
+; SI-NEXT: v_readlane_b32 s7, v44, 3
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: v_readlane_b32 s9, v44, 0
+; SI-NEXT: v_readlane_b32 s9, v44, 2
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_lshl_b32 s9, s9, 24
-; SI-NEXT: s_or_b32 s57, s9, s7
-; SI-NEXT: s_and_b32 s7, s28, 0xff
-; SI-NEXT: s_lshl_b32 s9, s82, 8
+; SI-NEXT: s_or_b32 s76, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v44, 0
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s9, s29, 8
; SI-NEXT: s_or_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s90, 0xff
+; SI-NEXT: s_and_b32 s9, s79, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s11, s54, 24
+; SI-NEXT: s_lshl_b32 s11, s23, 24
; SI-NEXT: s_or_b32 s77, s11, s9
; SI-NEXT: s_and_b32 s9, s91, 0xff
-; SI-NEXT: s_lshl_b32 s11, s92, 8
+; SI-NEXT: s_lshl_b32 s11, s16, 8
; SI-NEXT: s_or_b32 s9, s9, s11
-; SI-NEXT: s_and_b32 s11, s89, 0xff
-; SI-NEXT: s_mov_b32 s72, s44
+; SI-NEXT: s_and_b32 s11, s90, 0xff
+; SI-NEXT: s_mov_b32 s24, s85
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s44, s20, 24
-; SI-NEXT: s_mov_b32 s59, s16
-; SI-NEXT: s_or_b32 vcc_lo, s44, s11
-; SI-NEXT: s_and_b32 s11, s35, 0xff
-; SI-NEXT: s_lshl_b32 s44, s48, 8
-; SI-NEXT: v_readlane_b32 s16, v44, 4
+; SI-NEXT: s_mov_b32 s85, s26
+; SI-NEXT: s_mov_b32 s26, s44
+; SI-NEXT: s_lshl_b32 s44, s89, 24
+; SI-NEXT: v_writelane_b32 v44, s55, 10
+; SI-NEXT: s_mov_b32 s55, s34
+; SI-NEXT: s_mov_b32 s34, s23
+; SI-NEXT: s_or_b32 s23, s44, s11
+; SI-NEXT: s_and_b32 s11, s39, 0xff
+; SI-NEXT: s_lshl_b32 s44, s37, 8
; SI-NEXT: s_or_b32 s11, s11, s44
-; SI-NEXT: s_and_b32 s44, s16, 0xff
+; SI-NEXT: s_and_b32 s44, s31, 0xff
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s45, s95, 24
-; SI-NEXT: s_or_b32 vcc_hi, s45, s44
-; SI-NEXT: s_and_b32 s44, s51, 0xff
-; SI-NEXT: s_lshl_b32 s45, s65, 8
+; SI-NEXT: s_lshl_b32 s45, s86, 24
+; SI-NEXT: v_writelane_b32 v44, s67, 11
+; SI-NEXT: v_writelane_b32 v44, s50, 12
+; SI-NEXT: s_mov_b32 s50, s21
+; SI-NEXT: s_or_b32 s21, s45, s44
+; SI-NEXT: s_and_b32 s44, s94, 0xff
+; SI-NEXT: s_lshl_b32 s45, s49, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s50, 0xff
+; SI-NEXT: s_and_b32 s45, s68, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s39, 24
+; SI-NEXT: s_lshl_b32 s46, s38, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s67, s20
+; SI-NEXT: s_mov_b32 s95, s20
; SI-NEXT: s_or_b32 s20, s46, s45
; SI-NEXT: s_or_b32 s61, s44, s20
-; SI-NEXT: s_and_b32 s44, s64, 0xff
-; SI-NEXT: s_lshl_b32 s45, s81, 8
+; SI-NEXT: s_and_b32 s44, s69, 0xff
+; SI-NEXT: s_lshl_b32 s45, s83, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s98, 0xff
+; SI-NEXT: s_and_b32 s45, vcc_hi, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s71, 24
+; SI-NEXT: s_lshl_b32 s46, vcc_lo, 24
+; SI-NEXT: s_mov_b32 s52, s35
+; SI-NEXT: s_mov_b32 s35, s24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s70, s39
-; SI-NEXT: s_mov_b32 s39, s91
-; SI-NEXT: s_mov_b32 s91, s82
-; SI-NEXT: s_mov_b32 s82, s88
-; SI-NEXT: s_mov_b32 s88, s18
+; SI-NEXT: s_mov_b32 s24, s69
+; SI-NEXT: s_mov_b32 s69, s53
+; SI-NEXT: s_mov_b32 s53, s31
+; SI-NEXT: s_mov_b32 s31, s96
+; SI-NEXT: s_mov_b32 s96, s18
; SI-NEXT: s_or_b32 s18, s46, s45
; SI-NEXT: s_or_b32 s63, s44, s18
-; SI-NEXT: s_and_b32 s44, s83, 0xff
-; SI-NEXT: s_lshl_b32 s45, s87, 8
+; SI-NEXT: s_and_b32 s44, s56, 0xff
+; SI-NEXT: s_lshl_b32 s45, s78, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s23, 0xff
+; SI-NEXT: s_and_b32 s45, s70, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s22, 24
-; SI-NEXT: s_mov_b32 s80, s31
-; SI-NEXT: s_mov_b32 s31, s53
-; SI-NEXT: s_mov_b32 s53, s38
-; SI-NEXT: s_mov_b32 s38, s90
+; SI-NEXT: s_lshl_b32 s46, s99, 24
+; SI-NEXT: s_mov_b32 s56, s22
+; SI-NEXT: s_mov_b32 s22, s87
+; SI-NEXT: s_mov_b32 s87, s83
+; SI-NEXT: s_mov_b32 s83, s54
+; SI-NEXT: s_mov_b32 s54, s39
+; SI-NEXT: s_mov_b32 s39, s90
; SI-NEXT: s_mov_b32 s90, s19
; SI-NEXT: s_or_b32 s19, s46, s45
-; SI-NEXT: s_and_b32 s45, s84, 0xff
+; SI-NEXT: s_and_b32 s45, s57, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s47, 24
+; SI-NEXT: s_lshl_b32 s46, s59, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s97, s22
-; SI-NEXT: s_mov_b32 s22, s55
-; SI-NEXT: s_mov_b32 s55, s49
-; SI-NEXT: s_mov_b32 s49, s92
-; SI-NEXT: s_mov_b32 s92, s72
; SI-NEXT: s_or_b32 s72, s46, s45
; SI-NEXT: s_or_b32 s73, s44, s19
-; SI-NEXT: s_and_b32 s44, s99, 0xff
-; SI-NEXT: s_lshl_b32 s45, s58, 8
+; SI-NEXT: s_and_b32 s44, s30, 0xff
+; SI-NEXT: s_lshl_b32 s45, s47, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s86, 0xff
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_and_b32 s45, s84, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s56, 24
-; SI-NEXT: s_or_b32 s5, s5, s57
-; SI-NEXT: s_mov_b32 s76, s56
-; SI-NEXT: s_mov_b32 s56, s86
-; SI-NEXT: s_mov_b32 s86, s81
-; SI-NEXT: s_mov_b32 s81, s65
-; SI-NEXT: s_mov_b32 s65, s48
-; SI-NEXT: s_mov_b32 s48, s89
+; SI-NEXT: s_lshl_b32 s46, s58, 24
+; SI-NEXT: s_mov_b32 s97, s70
+; SI-NEXT: s_mov_b32 s70, s66
+; SI-NEXT: s_mov_b32 s66, s36
+; SI-NEXT: s_mov_b32 s36, s89
; SI-NEXT: s_mov_b32 s89, s17
; SI-NEXT: s_or_b32 s17, s46, s45
-; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: v_writelane_b32 v44, s68, 11
-; SI-NEXT: s_mov_b32 s94, s51
-; SI-NEXT: s_mov_b32 s51, s95
-; SI-NEXT: s_mov_b32 s95, s27
-; SI-NEXT: s_or_b32 s75, s44, s17
-; SI-NEXT: s_mov_b32 s44, s93
-; SI-NEXT: s_mov_b32 s93, s28
-; SI-NEXT: s_and_b32 s28, s42, 0xffff
-; SI-NEXT: s_and_b32 s27, s43, 0xffff
-; SI-NEXT: s_or_b32 s42, s12, s4
-; SI-NEXT: s_mov_b32 s43, s5
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_or_b32 s9, s9, vcc_lo
-; SI-NEXT: s_and_b32 s45, s78, 0xff
-; SI-NEXT: v_writelane_b32 v44, s4, 7
+; SI-NEXT: s_and_b32 s45, s98, 0xff
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s24, 24
-; SI-NEXT: v_writelane_b32 v44, s5, 8
-; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 16
+; SI-NEXT: s_lshl_b32 s46, s25, 24
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_or_b32 s5, s5, s76
; SI-NEXT: s_or_b32 s7, s7, s77
-; SI-NEXT: s_or_b32 s11, s11, vcc_hi
-; SI-NEXT: s_mov_b32 s68, s34
+; SI-NEXT: s_or_b32 s9, s9, s23
+; SI-NEXT: s_mov_b32 s67, s27
+; SI-NEXT: s_or_b32 s11, s11, s21
+; SI-NEXT: s_mov_b32 s82, s65
+; SI-NEXT: s_mov_b32 s65, s38
+; SI-NEXT: s_mov_b32 s38, s91
+; SI-NEXT: s_mov_b32 s91, s29
; SI-NEXT: s_or_b32 s74, s46, s45
+; SI-NEXT: s_or_b32 s75, s44, s17
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_and_b32 s58, s15, 0xffff
+; SI-NEXT: s_mov_b32 s48, s16
; SI-NEXT: s_and_b32 s16, s40, 0xffff
-; SI-NEXT: s_mov_b32 s34, s29
; SI-NEXT: s_and_b32 s29, s41, 0xffff
-; SI-NEXT: v_writelane_b32 v44, s4, 9
-; SI-NEXT: s_mov_b32 s45, s78
-; SI-NEXT: s_mov_b32 s78, s84
-; SI-NEXT: s_mov_b32 s84, s69
-; SI-NEXT: s_mov_b32 s69, s59
+; SI-NEXT: s_mov_b32 s45, s98
+; SI-NEXT: s_mov_b32 s98, s28
+; SI-NEXT: s_and_b32 s28, s42, 0xffff
+; SI-NEXT: s_and_b32 s27, s43, 0xffff
+; SI-NEXT: s_mov_b32 s44, s93
+; SI-NEXT: s_or_b32 s42, s12, s4
+; SI-NEXT: s_mov_b32 s43, s5
+; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
; SI-NEXT: s_or_b32 s40, s13, s6
; SI-NEXT: s_mov_b32 s41, s7
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
; SI-NEXT: s_or_b32 s14, s14, s8
; SI-NEXT: s_mov_b32 s15, s9
-; SI-NEXT: v_writelane_b32 v44, s5, 10
+; SI-NEXT: s_lshr_b64 s[92:93], s[8:9], 16
; SI-NEXT: s_or_b32 s12, s58, s10
; SI-NEXT: s_mov_b32 s13, s11
; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
@@ -69996,63 +71016,63 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s4, s27, s74
; SI-NEXT: s_mov_b32 s5, s75
; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
-; SI-NEXT: s_mov_b32 s29, s34
-; SI-NEXT: s_mov_b32 s16, s69
-; SI-NEXT: s_mov_b32 s69, s84
-; SI-NEXT: s_mov_b32 s84, s78
-; SI-NEXT: s_mov_b32 s78, s45
-; SI-NEXT: s_mov_b32 s28, s93
+; SI-NEXT: s_mov_b32 s16, s48
; SI-NEXT: s_mov_b32 s93, s44
-; SI-NEXT: s_lshr_b32 s47, s57, 16
+; SI-NEXT: s_mov_b32 s44, s26
+; SI-NEXT: s_mov_b32 s26, s85
+; SI-NEXT: s_mov_b32 s28, s98
+; SI-NEXT: s_mov_b32 s98, s45
+; SI-NEXT: s_lshr_b32 s47, s76, 16
; SI-NEXT: s_lshr_b32 s57, s77, 16
-; SI-NEXT: s_lshr_b32 s59, vcc_lo, 16
-; SI-NEXT: s_lshr_b32 s61, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s59, s23, 16
+; SI-NEXT: s_mov_b32 s23, s34
+; SI-NEXT: s_mov_b32 s34, s55
+; SI-NEXT: v_readlane_b32 s55, v44, 10
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_mov_b32 s21, s50
+; SI-NEXT: s_mov_b32 s27, s67
+; SI-NEXT: v_readlane_b32 s50, v44, 12
+; SI-NEXT: v_readlane_b32 s67, v44, 11
; SI-NEXT: s_lshr_b32 s63, s20, 16
-; SI-NEXT: s_mov_b32 s20, s67
-; SI-NEXT: s_mov_b32 s34, s68
-; SI-NEXT: v_readlane_b32 s68, v44, 11
+; SI-NEXT: s_mov_b32 s20, s95
+; SI-NEXT: s_mov_b32 s85, s35
+; SI-NEXT: s_mov_b32 s35, s52
; SI-NEXT: s_lshr_b32 s73, s18, 16
-; SI-NEXT: s_mov_b32 s18, s88
-; SI-NEXT: s_mov_b32 s27, s95
-; SI-NEXT: s_mov_b32 s95, s51
-; SI-NEXT: s_mov_b32 s51, s94
-; SI-NEXT: s_mov_b32 s88, s82
-; SI-NEXT: s_mov_b32 s82, s91
-; SI-NEXT: s_mov_b32 s91, s39
-; SI-NEXT: s_mov_b32 s39, s70
+; SI-NEXT: s_mov_b32 s18, s96
+; SI-NEXT: s_mov_b32 s96, s31
+; SI-NEXT: s_mov_b32 s31, s53
+; SI-NEXT: s_mov_b32 s53, s69
+; SI-NEXT: s_mov_b32 s69, s24
+; SI-NEXT: s_mov_b32 s29, s91
+; SI-NEXT: s_mov_b32 s91, s38
+; SI-NEXT: s_mov_b32 s38, s65
+; SI-NEXT: s_mov_b32 s65, s82
; SI-NEXT: s_lshr_b32 s75, s19, 16
; SI-NEXT: s_mov_b32 s19, s90
-; SI-NEXT: s_mov_b32 s90, s38
-; SI-NEXT: s_mov_b32 s38, s53
-; SI-NEXT: s_mov_b32 s53, s31
-; SI-NEXT: s_mov_b32 s44, s92
-; SI-NEXT: s_mov_b32 s92, s49
-; SI-NEXT: s_mov_b32 s49, s55
-; SI-NEXT: s_mov_b32 s55, s22
-; SI-NEXT: s_mov_b32 s31, s80
-; SI-NEXT: s_mov_b32 s22, s97
+; SI-NEXT: s_mov_b32 s90, s39
+; SI-NEXT: s_mov_b32 s39, s54
+; SI-NEXT: s_mov_b32 s54, s83
+; SI-NEXT: s_mov_b32 s83, s87
+; SI-NEXT: s_mov_b32 s87, s22
+; SI-NEXT: s_mov_b32 s22, s56
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_mov_b32 s17, s89
-; SI-NEXT: s_mov_b32 s89, s48
-; SI-NEXT: s_mov_b32 s48, s65
-; SI-NEXT: s_mov_b32 s65, s81
-; SI-NEXT: s_mov_b32 s81, s86
-; SI-NEXT: s_mov_b32 s86, s56
-; SI-NEXT: s_mov_b32 s56, s76
+; SI-NEXT: s_mov_b32 s89, s36
+; SI-NEXT: s_mov_b32 s36, s66
+; SI-NEXT: s_mov_b32 s66, s70
+; SI-NEXT: s_mov_b32 s70, s97
; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB99_3
; SI-NEXT: .LBB99_2:
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: s_mov_b64 s[76:77], -1
-; SI-NEXT: v_writelane_b32 v44, s4, 7
-; SI-NEXT: v_writelane_b32 v44, s5, 8
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr47
; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr58
@@ -70066,215 +71086,216 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: v_writelane_b32 v44, s4, 9
-; SI-NEXT: v_writelane_b32 v44, s5, 10
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: .LBB99_3: ; %Flow
-; SI-NEXT: s_andn2_b64 vcc, exec, s[76:77]
-; SI-NEXT: v_readlane_b32 s76, v44, 7
-; SI-NEXT: v_readlane_b32 s77, v44, 8
-; SI-NEXT: s_cbranch_vccnz .LBB99_5
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s76, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s76, 1
+; SI-NEXT: s_cbranch_scc1 .LBB99_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s5, s25, 8
-; SI-NEXT: s_add_i32 s78, s78, 3
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s4, s22, 0xff
+; SI-NEXT: s_lshl_b32 s5, s26, 8
+; SI-NEXT: s_add_i32 s98, s98, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s78, 0xff
+; SI-NEXT: s_and_b32 s6, s98, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s24, 24
+; SI-NEXT: s_lshl_b32 s5, s25, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s99, s99, 3
-; SI-NEXT: v_readlane_b32 s6, v44, 6
+; SI-NEXT: s_add_i32 s97, s30, 3
+; SI-NEXT: v_readlane_b32 s6, v44, 9
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s99, 0xff
+; SI-NEXT: s_and_b32 s5, s97, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s7, s86, 0xff
+; SI-NEXT: v_readlane_b32 s6, v44, 7
+; SI-NEXT: s_and_b32 s7, s84, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s6, s56, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s97, s79, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
+; SI-NEXT: v_readlane_b32 s8, v44, 6
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s97, 0xff
-; SI-NEXT: s_lshl_b32 s7, s36, 8
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_and_b32 s6, s87, 0xff
+; SI-NEXT: s_lshl_b32 s7, s44, 8
+; SI-NEXT: s_add_i32 s82, s8, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v44, 5
-; SI-NEXT: s_and_b32 s8, s84, 0xff
+; SI-NEXT: v_readlane_b32 s7, v44, 8
+; SI-NEXT: s_and_b32 s8, s82, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_add_i32 s83, s83, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s83, 0xff
-; SI-NEXT: s_lshl_b32 s8, s87, 8
-; SI-NEXT: s_add_i32 s80, s23, 3
+; SI-NEXT: v_readlane_b32 s7, v44, 5
+; SI-NEXT: s_add_i32 s81, s7, 3
+; SI-NEXT: s_and_b32 s7, s81, 0xff
+; SI-NEXT: s_lshl_b32 s8, s78, 8
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s9, s80, 0xff
+; SI-NEXT: s_and_b32 s9, s70, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s8, s22, 24
+; SI-NEXT: s_lshl_b32 s8, s99, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s67, s67, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s68, 0xff
-; SI-NEXT: s_lshl_b32 s9, s69, 8
-; SI-NEXT: s_add_i32 s66, s66, 3
+; SI-NEXT: s_and_b32 s8, s67, 0xff
+; SI-NEXT: s_lshl_b32 s9, s71, 8
+; SI-NEXT: s_add_i32 s65, s65, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s10, s66, 0xff
+; SI-NEXT: s_and_b32 s10, s65, 0xff
; SI-NEXT: s_addk_i32 s8, 0x300
; SI-NEXT: s_lshl_b32 s9, s55, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s64, s64, 3
+; SI-NEXT: s_add_i32 s55, s69, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s64, 0xff
-; SI-NEXT: s_lshl_b32 s10, s81, 8
-; SI-NEXT: s_add_i32 s69, s98, 3
+; SI-NEXT: s_and_b32 s9, s55, 0xff
+; SI-NEXT: s_lshl_b32 s10, s83, 8
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s11, s69, 0xff
+; SI-NEXT: s_and_b32 s11, vcc_hi, 0xff
; SI-NEXT: s_addk_i32 s9, 0x300
-; SI-NEXT: s_lshl_b32 s10, s71, 24
+; SI-NEXT: s_lshl_b32 s10, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_add_i32 s55, s31, 3
+; SI-NEXT: s_add_i32 s54, s54, 3
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s55, 0xff
-; SI-NEXT: s_lshl_b32 s11, s88, 8
-; SI-NEXT: s_add_i32 s53, s53, 3
+; SI-NEXT: s_and_b32 s10, s54, 0xff
+; SI-NEXT: s_lshl_b32 s11, s66, 8
+; SI-NEXT: s_add_i32 s52, s88, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s12, s53, 0xff
+; SI-NEXT: s_and_b32 s12, s52, 0xff
; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s11, s37, 24
+; SI-NEXT: s_lshl_b32 s11, s50, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_add_i32 s51, s51, 3
+; SI-NEXT: s_add_i32 s50, s94, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s51, 0xff
-; SI-NEXT: s_lshl_b32 s12, s65, 8
+; SI-NEXT: s_and_b32 s11, s50, 0xff
+; SI-NEXT: s_lshl_b32 s12, s49, 8
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_lshl_b32 s12, s39, 24
-; SI-NEXT: s_add_i32 s39, s50, 3
-; SI-NEXT: s_and_b32 s13, s39, 0xff
+; SI-NEXT: s_lshl_b32 s12, s38, 24
+; SI-NEXT: s_add_i32 s38, s68, 3
+; SI-NEXT: s_and_b32 s13, s38, 0xff
; SI-NEXT: s_addk_i32 s11, 0x300
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: s_add_i32 s50, s96, 3
+; SI-NEXT: s_add_i32 s49, s64, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s50, 0xff
-; SI-NEXT: s_lshl_b32 s13, s16, 8
-; SI-NEXT: s_add_i32 s37, s52, 3
+; SI-NEXT: s_and_b32 s12, s49, 0xff
+; SI-NEXT: s_lshl_b32 s13, s53, 8
+; SI-NEXT: s_add_i32 s36, s36, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s14, s37, 0xff
+; SI-NEXT: s_and_b32 s14, s36, 0xff
; SI-NEXT: s_addk_i32 s12, 0x300
; SI-NEXT: s_lshl_b32 s13, s34, 24
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s34, s39, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s35, 0xff
-; SI-NEXT: s_lshl_b32 s14, s48, 8
-; SI-NEXT: v_readlane_b32 s15, v44, 4
+; SI-NEXT: s_and_b32 s13, s34, 0xff
+; SI-NEXT: s_lshl_b32 s14, s37, 8
+; SI-NEXT: s_add_i32 s95, s31, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_lshl_b32 s14, s95, 24
-; SI-NEXT: s_add_i32 s95, s15, 3
; SI-NEXT: s_and_b32 s15, s95, 0xff
; SI-NEXT: s_addk_i32 s13, 0x300
+; SI-NEXT: s_lshl_b32 s14, s86, 24
; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: s_add_i32 s34, s49, 3
+; SI-NEXT: s_add_i32 vcc_lo, s51, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s34, 0xff
-; SI-NEXT: s_lshl_b32 s15, s38, 8
+; SI-NEXT: s_and_b32 s14, vcc_lo, 0xff
+; SI-NEXT: s_lshl_b32 s15, s35, 8
+; SI-NEXT: s_add_i32 s93, s93, 3
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_lshl_b32 s15, s93, 24
-; SI-NEXT: s_add_i32 s93, s30, 3
-; SI-NEXT: s_and_b32 s21, s93, 0xff
+; SI-NEXT: s_and_b32 s22, s93, 0xff
; SI-NEXT: s_addk_i32 s14, 0x300
-; SI-NEXT: s_lshl_b32 s21, s21, 16
+; SI-NEXT: s_lshl_b32 s15, s28, 24
+; SI-NEXT: s_lshl_b32 s22, s22, 16
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_or_b32 s15, s15, s21
+; SI-NEXT: s_or_b32 s15, s15, s22
; SI-NEXT: s_add_i32 s91, s91, 3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s15, s91, 0xff
-; SI-NEXT: s_lshl_b32 s21, s92, 8
-; SI-NEXT: s_add_i32 s89, s89, 3
-; SI-NEXT: s_or_b32 s15, s21, s15
-; SI-NEXT: s_and_b32 s22, s89, 0xff
+; SI-NEXT: s_lshl_b32 s22, s16, 8
+; SI-NEXT: s_or_b32 s15, s22, s15
+; SI-NEXT: s_lshl_b32 s22, s89, 24
+; SI-NEXT: s_add_i32 s89, s90, 3
+; SI-NEXT: s_and_b32 s40, s89, 0xff
; SI-NEXT: s_addk_i32 s15, 0x300
-; SI-NEXT: s_lshl_b32 s21, s20, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_add_i32 s16, s44, 3
+; SI-NEXT: s_lshl_b32 s40, s40, 16
+; SI-NEXT: v_readlane_b32 s16, v44, 0
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_or_b32 s21, s21, s22
-; SI-NEXT: s_add_i32 s24, s29, 3
+; SI-NEXT: s_or_b32 s22, s22, s40
+; SI-NEXT: s_add_i32 s24, s85, 3
+; SI-NEXT: v_readlane_b32 s25, v44, 1
+; SI-NEXT: s_add_i32 s28, s16, 3
+; SI-NEXT: v_readlane_b32 s16, v44, 4
+; SI-NEXT: s_or_b32 s15, s22, s15
+; SI-NEXT: s_and_b32 s22, s24, 0xff
+; SI-NEXT: s_lshl_b32 s24, s27, 8
+; SI-NEXT: s_add_i32 s26, s25, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_and_b32 s25, s26, 0xff
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s15, s21, s15
-; SI-NEXT: s_and_b32 s21, s24, 0xff
-; SI-NEXT: s_lshl_b32 s22, s85, 8
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_addk_i32 s22, 0x300
+; SI-NEXT: s_lshl_b32 s24, s96, 24
+; SI-NEXT: s_lshl_b32 s25, s25, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_and_b32 s20, s26, 0xff
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s19, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_addk_i32 s21, 0x300
-; SI-NEXT: s_lshl_b32 s22, s27, 24
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_or_b32 s22, s24, s22
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_or_b32 s22, s22, s20
+; SI-NEXT: s_add_i32 s40, s22, 0x3000000
+; SI-NEXT: s_and_b32 s22, s28, 0xff
+; SI-NEXT: s_lshl_b32 s24, s29, 8
+; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: v_readlane_b32 s18, v44, 3
+; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_and_b32 s24, s79, 0xff
; SI-NEXT: s_add_i32 s42, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v44, 3
-; SI-NEXT: s_add_i32 s40, s21, 0x3000000
-; SI-NEXT: s_and_b32 s21, s28, 0xff
-; SI-NEXT: s_lshl_b32 s22, s82, 8
-; SI-NEXT: s_add_i32 s79, s90, 3
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v44, 2
-; SI-NEXT: v_readlane_b32 s18, v44, 1
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_and_b32 s22, s79, 0xff
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_and_b32 s16, s20, 0xff
+; SI-NEXT: s_lshl_b32 s17, s21, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_addk_i32 s21, 0x300
-; SI-NEXT: s_lshl_b32 s20, s54, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_addk_i32 s22, 0x300
+; SI-NEXT: s_lshl_b32 s23, s23, 24
+; SI-NEXT: s_lshl_b32 s24, s24, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v44, 0
+; SI-NEXT: v_readlane_b32 s17, v44, 2
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_or_b32 s20, s20, s22
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_or_b32 s22, s23, s22
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
@@ -70287,20 +71308,19 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
+; SI-NEXT: s_add_i32 s41, s22, 0x3000000
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s14, s14, 0x3000000
; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: s_add_i32 s41, s20, 0x3000000
-; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s43, s16, 0x3000000
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v44, s16, 9
+; SI-NEXT: s_lshr_b64 s[80:81], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
; SI-NEXT: s_lshr_b32 s47, s43, 16
; SI-NEXT: s_lshr_b32 s57, s41, 16
; SI-NEXT: s_lshr_b32 s59, s15, 16
@@ -70309,13 +71329,12 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s73, s9, 16
; SI-NEXT: s_lshr_b32 s75, s7, 16
; SI-NEXT: s_lshr_b32 s45, s5, 16
-; SI-NEXT: v_writelane_b32 v44, s17, 10
; SI-NEXT: .LBB99_5: ; %end
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s16, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s76, 16
+; SI-NEXT: s_lshl_b32 s17, s80, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s43, 0xffff
; SI-NEXT: s_lshl_b32 s18, s47, 16
@@ -70326,9 +71345,8 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s19, s41, 0xffff
; SI-NEXT: s_lshl_b32 s20, s57, 16
; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: v_readlane_b32 s20, v44, 9
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_lshl_b32 s20, s92, 16
; SI-NEXT: s_or_b32 s14, s14, s20
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_lshl_b32 s20, s59, 16
@@ -70364,7 +71382,6 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s20, s45, 16
; SI-NEXT: s_or_b32 s5, s5, s20
; SI-NEXT: v_readlane_b32 s30, v43, 34
-; SI-NEXT: v_readlane_b32 s21, v44, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -70480,7 +71497,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB99_4
+; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -70572,8 +71589,17 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v33, v28, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB99_3
-; VI-NEXT: .LBB99_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB99_3
+; VI-NEXT: .LBB99_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB99_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB99_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_u32_e32 v0, vcc, 3, v34
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v8, v0, v31, s4
@@ -70733,12 +71759,9 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB99_3: ; %end
+; VI-NEXT: .LBB99_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB99_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB99_2
;
; GFX9-LABEL: bitcast_v64i8_to_v32i16_scalar:
; GFX9: ; %bb.0:
@@ -70797,7 +71820,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
@@ -70903,8 +71926,17 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB99_3
-; GFX9-NEXT: .LBB99_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB99_3
+; GFX9-NEXT: .LBB99_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB99_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -71061,12 +72093,9 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
-; GFX9-NEXT: .LBB99_3: ; %end
+; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB99_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB99_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32i16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -71123,7 +72152,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
@@ -71202,9 +72231,17 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v18.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-TRUE16-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB99_3
+; GFX11-TRUE16-NEXT: .LBB99_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -71321,12 +72358,9 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: .LBB99_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB99_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB99_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v32i16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -71383,7 +72417,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s58, 0xff
@@ -71476,9 +72510,17 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v16, 16, v17
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, s77
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-FAKE16-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB99_3
+; GFX11-FAKE16-NEXT: .LBB99_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -71611,12 +72653,9 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v16, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v14, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v22
-; GFX11-FAKE16-NEXT: .LBB99_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB99_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB99_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -72139,10 +73178,8 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: v_writelane_b32 v40, s35, 1
; SI-NEXT: v_writelane_b32 v40, s36, 2
; SI-NEXT: v_writelane_b32 v40, s37, 3
-; SI-NEXT: v_writelane_b32 v40, s38, 4
-; SI-NEXT: v_writelane_b32 v40, s39, 5
-; SI-NEXT: v_writelane_b32 v40, s30, 6
-; SI-NEXT: v_writelane_b32 v40, s31, 7
+; SI-NEXT: v_writelane_b32 v40, s30, 4
+; SI-NEXT: v_writelane_b32 v40, s31, 5
; SI-NEXT: v_readfirstlane_b32 s60, v1
; SI-NEXT: v_readfirstlane_b32 s56, v0
; SI-NEXT: s_lshr_b32 s43, s29, 16
@@ -72163,7 +73200,7 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: s_lshr_b32 s59, s56, 16
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB101_3
+; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s44, s16, 16
; SI-NEXT: s_lshl_b32 s45, s6, 16
@@ -72189,16 +73226,56 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: s_lshl_b32 s93, s40, 16
; SI-NEXT: s_lshl_b32 s94, s27, 16
; SI-NEXT: s_lshl_b32 s95, s41, 16
-; SI-NEXT: s_lshl_b32 s30, s28, 16
-; SI-NEXT: s_lshl_b32 s31, s42, 16
-; SI-NEXT: s_lshl_b32 s34, s29, 16
-; SI-NEXT: s_lshl_b32 s35, s43, 16
-; SI-NEXT: s_lshl_b32 s36, s56, 16
-; SI-NEXT: s_lshl_b32 s37, s59, 16
-; SI-NEXT: s_lshl_b32 s38, s60, 16
-; SI-NEXT: s_lshl_b32 s39, s63, 16
-; SI-NEXT: s_cbranch_execnz .LBB101_4
-; SI-NEXT: .LBB101_2: ; %cmp.true
+; SI-NEXT: s_lshl_b32 vcc_lo, s28, 16
+; SI-NEXT: s_lshl_b32 vcc_hi, s42, 16
+; SI-NEXT: s_lshl_b32 s30, s29, 16
+; SI-NEXT: s_lshl_b32 s31, s43, 16
+; SI-NEXT: s_lshl_b32 s34, s56, 16
+; SI-NEXT: s_lshl_b32 s35, s59, 16
+; SI-NEXT: s_lshl_b32 s36, s60, 16
+; SI-NEXT: s_lshl_b32 s37, s63, 16
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s60
; SI-NEXT: v_cvt_f32_f16_e32 v2, s59
@@ -72327,50 +73404,16 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: s_branch .LBB101_5
-; SI-NEXT: .LBB101_3:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: s_branch .LBB101_2
-; SI-NEXT: .LBB101_4:
-; SI-NEXT: v_mov_b32_e32 v1, s39
-; SI-NEXT: v_mov_b32_e32 v0, s38
-; SI-NEXT: v_mov_b32_e32 v3, s37
-; SI-NEXT: v_mov_b32_e32 v2, s36
-; SI-NEXT: v_mov_b32_e32 v5, s35
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_mov_b32_e32 v7, s31
-; SI-NEXT: v_mov_b32_e32 v6, s30
+; SI-NEXT: s_branch .LBB101_6
+; SI-NEXT: .LBB101_5:
+; SI-NEXT: v_mov_b32_e32 v1, s37
+; SI-NEXT: v_mov_b32_e32 v0, s36
+; SI-NEXT: v_mov_b32_e32 v3, s35
+; SI-NEXT: v_mov_b32_e32 v2, s34
+; SI-NEXT: v_mov_b32_e32 v5, s31
+; SI-NEXT: v_mov_b32_e32 v4, s30
+; SI-NEXT: v_mov_b32_e32 v7, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v6, vcc_lo
; SI-NEXT: v_mov_b32_e32 v9, s95
; SI-NEXT: v_mov_b32_e32 v8, s94
; SI-NEXT: v_mov_b32_e32 v11, s93
@@ -72395,7 +73438,7 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: v_mov_b32_e32 v28, s46
; SI-NEXT: v_mov_b32_e32 v31, s45
; SI-NEXT: v_mov_b32_e32 v30, s44
-; SI-NEXT: .LBB101_5: ; %end
+; SI-NEXT: .LBB101_6: ; %end
; SI-NEXT: v_mul_f32_e32 v31, 1.0, v31
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; SI-NEXT: v_mul_f32_e32 v30, 1.0, v30
@@ -72459,11 +73502,9 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; SI-NEXT: v_lshr_b64 v[12:13], v[36:37], 16
; SI-NEXT: v_lshr_b64 v[13:14], v[38:39], 16
; SI-NEXT: v_lshr_b64 v[14:15], v[48:49], 16
-; SI-NEXT: v_readlane_b32 s30, v40, 6
+; SI-NEXT: v_readlane_b32 s30, v40, 4
; SI-NEXT: v_lshr_b64 v[15:16], v[50:51], 16
-; SI-NEXT: v_readlane_b32 s31, v40, 7
-; SI-NEXT: v_readlane_b32 s39, v40, 5
-; SI-NEXT: v_readlane_b32 s38, v40, 4
+; SI-NEXT: v_readlane_b32 s31, v40, 5
; SI-NEXT: v_readlane_b32 s37, v40, 3
; SI-NEXT: v_readlane_b32 s36, v40, 2
; SI-NEXT: v_readlane_b32 s35, v40, 1
@@ -72486,10 +73527,18 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB101_3
+; VI-NEXT: s_cbranch_scc0 .LBB101_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB101_4
-; VI-NEXT: .LBB101_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB101_3
+; VI-NEXT: .LBB101_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB101_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB101_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s30, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_mov_b32_e32 v3, s5
@@ -72571,10 +73620,8 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; VI-NEXT: v_add_f16_e32 v1, s17, v1
; VI-NEXT: v_or_b32_e32 v1, v1, v17
; VI-NEXT: v_or_b32_e32 v0, v0, v16
-; VI-NEXT: s_branch .LBB101_5
-; VI-NEXT: .LBB101_3:
-; VI-NEXT: s_branch .LBB101_2
-; VI-NEXT: .LBB101_4:
+; VI-NEXT: s_branch .LBB101_6
+; VI-NEXT: .LBB101_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -72591,7 +73638,7 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB101_5: ; %end
+; VI-NEXT: .LBB101_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v18, 0
; VI-NEXT: v_readlane_b32 s31, v18, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -72612,10 +73659,18 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB101_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB101_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB101_4
-; GFX9-NEXT: .LBB101_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB101_3
+; GFX9-NEXT: .LBB101_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB101_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB101_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s31, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s30, v0 op_sel_hi:[1,0]
@@ -72633,10 +73688,8 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB101_5
-; GFX9-NEXT: .LBB101_3:
-; GFX9-NEXT: s_branch .LBB101_2
-; GFX9-NEXT: .LBB101_4:
+; GFX9-NEXT: s_branch .LBB101_6
+; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -72653,7 +73706,7 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB101_5: ; %end
+; GFX9-NEXT: .LBB101_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v16, 0
; GFX9-NEXT: v_readlane_b32 s31, v16, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -72671,11 +73724,16 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; GFX11-NEXT: s_mov_b32 s12, s0
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB101_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v13, 0x200, s25 op_sel_hi:[0,1]
@@ -72693,8 +73751,6 @@ define inreg <32 x bfloat> @bitcast_v32f16_to_v32bf16_scalar(<32 x half> inreg %
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s13 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s12 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB101_3:
-; GFX11-NEXT: s_branch .LBB101_2
; GFX11-NEXT: .LBB101_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -74333,7 +75389,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v53, 1.0, s44
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: s_cbranch_scc0 .LBB103_4
+; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
@@ -74371,6 +75427,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v40
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v47
; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
@@ -74491,272 +75548,15 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v43, v6
; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: s_cbranch_execnz .LBB103_3
-; SI-NEXT: .LBB103_2: ; %cmp.true
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v43
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshr_b64 v[52:53], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v47
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; SI-NEXT: v_lshr_b64 v[17:18], v[1:2], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v28
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v42
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v41
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v2
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
-; SI-NEXT: v_lshr_b64 v[55:56], v[1:2], 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 16
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; SI-NEXT: v_mov_b32_e32 v1, v12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v7
-; SI-NEXT: v_lshr_b64 v[53:54], v[14:15], 16
-; SI-NEXT: v_mov_b32_e32 v54, v17
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v44
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v48
-; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[7:8], v[38:39], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[62:63], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[19:20], v[36:37], 16
-; SI-NEXT: v_lshr_b64 v[22:23], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[50:51], v[28:29], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: v_lshr_b64 v[1:2], v[60:61], 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: v_lshr_b64 v[57:58], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[11:12], v[56:57], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; SI-NEXT: v_mov_b32_e32 v2, v7
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[31:32], v[34:35], 16
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v2, v19
-; SI-NEXT: v_lshr_b64 v[24:25], v[30:31], 16
-; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[26:27], 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: .LBB103_3: ; %end
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v24
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v31
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v52
-; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v49
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v53
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v35
-; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v9
-; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v61
-; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
-; SI-NEXT: v_or_b32_e32 v5, v5, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v54
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v39
-; SI-NEXT: v_or_b32_e32 v7, v7, v8
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v18
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v29
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v37
-; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v50
-; SI-NEXT: v_or_b32_e32 v10, v10, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v63
-; SI-NEXT: v_or_b32_e32 v11, v11, v12
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v25
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_or_b32_e32 v12, v12, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v27
-; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v55
-; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB103_4:
+; SI-NEXT: s_branch .LBB103_3
+; SI-NEXT: .LBB103_2:
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v5, v32
; SI-NEXT: v_mov_b32_e32 v8, v18
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v16, v53
+; SI-NEXT: s_mov_b64 s[4:5], -1
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
@@ -74797,7 +75597,269 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: ; implicit-def: $vgpr29
; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: s_branch .LBB103_2
+; SI-NEXT: .LBB103_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB103_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v43
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v40
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_lshr_b64 v[52:53], v[1:2], 16
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v47
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v17
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v45
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; SI-NEXT: v_lshr_b64 v[17:18], v[1:2], 16
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v42
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v41
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
+; SI-NEXT: v_lshr_b64 v[55:56], v[1:2], 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; SI-NEXT: v_mov_b32_e32 v1, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[27:28], v[11:12], 16
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v7
+; SI-NEXT: v_lshr_b64 v[53:54], v[14:15], 16
+; SI-NEXT: v_mov_b32_e32 v54, v17
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v63, 16, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v36
+; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v20
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v38
+; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v48
+; SI-NEXT: v_add_f32_e32 v48, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[7:8], v[38:39], 16
+; SI-NEXT: v_lshr_b64 v[29:30], v[62:63], 16
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v3
+; SI-NEXT: v_lshr_b64 v[19:20], v[36:37], 16
+; SI-NEXT: v_lshr_b64 v[22:23], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[28:29], 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v60
+; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
+; SI-NEXT: v_lshr_b64 v[1:2], v[60:61], 16
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
+; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[57:58], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[56:57], 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v2
+; SI-NEXT: v_mov_b32_e32 v2, v7
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[31:32], v[34:35], 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, v19
+; SI-NEXT: v_lshr_b64 v[24:25], v[30:31], 16
+; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[25:26], v[26:27], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: .LBB103_5: ; %end
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v24
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v31
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v52
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v53
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v35
+; SI-NEXT: v_or_b32_e32 v1, v3, v1
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v9
+; SI-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v61
+; SI-NEXT: v_or_b32_e32 v3, v4, v3
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v11
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
+; SI-NEXT: v_or_b32_e32 v5, v5, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v54
+; SI-NEXT: v_or_b32_e32 v6, v6, v8
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v39
+; SI-NEXT: v_or_b32_e32 v7, v7, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v18
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT: v_or_b32_e32 v8, v8, v9
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v10
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v37
+; SI-NEXT: v_or_b32_e32 v9, v9, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v12
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v50
+; SI-NEXT: v_or_b32_e32 v10, v10, v12
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v63
+; SI-NEXT: v_or_b32_e32 v11, v11, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v25
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v13
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v27
+; SI-NEXT: v_or_b32_e32 v13, v13, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v55
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_or_b32_e32 v15, v15, v16
+; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: bitcast_v32bf16_to_v32f16_scalar:
; VI: ; %bb.0:
@@ -74811,10 +75873,18 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_readfirstlane_b32 s31, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s30, v0
-; VI-NEXT: s_cbranch_scc0 .LBB103_3
+; VI-NEXT: s_cbranch_scc0 .LBB103_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB103_4
-; VI-NEXT: .LBB103_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB103_3
+; VI-NEXT: .LBB103_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB103_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB103_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v16, 0x40c00000
; VI-NEXT: s_lshl_b32 s4, s26, 16
; VI-NEXT: v_add_f32_e32 v4, s4, v16
@@ -75111,10 +76181,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_lshrrev_b64 v[17:18], 16, v[17:18]
; VI-NEXT: v_lshrrev_b64 v[15:16], 16, v[15:16]
; VI-NEXT: v_mov_b32_e32 v13, v17
-; VI-NEXT: s_branch .LBB103_5
-; VI-NEXT: .LBB103_3:
-; VI-NEXT: s_branch .LBB103_2
-; VI-NEXT: .LBB103_4:
+; VI-NEXT: s_branch .LBB103_6
+; VI-NEXT: .LBB103_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -75131,7 +76199,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v15, s31
-; VI-NEXT: .LBB103_5: ; %end
+; VI-NEXT: .LBB103_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v20, 0
; VI-NEXT: v_readlane_b32 s31, v20, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -75152,10 +76220,18 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s31, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s30, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB103_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB103_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB103_4
-; GFX9-NEXT: .LBB103_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB103_3
+; GFX9-NEXT: .LBB103_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB103_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: s_and_b32 s5, s30, 0xffff0000
; GFX9-NEXT: v_add_f32_e32 v1, s5, v0
@@ -75462,10 +76538,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_cndmask_b32_e32 v17, v18, v19, vcc
; GFX9-NEXT: v_and_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
-; GFX9-NEXT: s_branch .LBB103_5
-; GFX9-NEXT: .LBB103_3:
-; GFX9-NEXT: s_branch .LBB103_2
-; GFX9-NEXT: .LBB103_4:
+; GFX9-NEXT: s_branch .LBB103_6
+; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -75482,7 +76556,7 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX9-NEXT: v_mov_b32_e32 v13, s29
; GFX9-NEXT: v_mov_b32_e32 v14, s30
; GFX9-NEXT: v_mov_b32_e32 v15, s31
-; GFX9-NEXT: .LBB103_5: ; %end
+; GFX9-NEXT: .LBB103_6: ; %end
; GFX9-NEXT: v_readlane_b32 s30, v20, 0
; GFX9-NEXT: v_readlane_b32 s31, v20, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -75500,11 +76574,16 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s12, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-TRUE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_and_b32 s0, s12, 0xffff0000
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s12, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -75807,8 +76886,6 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v28
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v31.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB103_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB103_2
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -75829,11 +76906,16 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_mov_b32 s12, s0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-FAKE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX11-FAKE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_and_b32 s0, s12, 0xffff0000
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s12, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -76160,8 +77242,6 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v23
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB103_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB103_2
; GFX11-FAKE16-NEXT: .LBB103_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s14 :: v_dual_mov_b32 v3, s15
@@ -78087,7 +79167,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: ; implicit-def: $vgpr19 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB105_3
+; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s69, 16
@@ -78186,8 +79266,73 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 24
; SI-NEXT: s_lshr_b64 s[34:35], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[36:37], s[4:5], 8
-; SI-NEXT: s_cbranch_execnz .LBB105_4
-; SI-NEXT: .LBB105_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 vcc, 0
+; SI-NEXT: s_branch .LBB105_3
+; SI-NEXT: .LBB105_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b64 vcc, -1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v19, s4, 0
+; SI-NEXT: v_writelane_b32 v19, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr57
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr47
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: v_writelane_b32 v19, s4, 2
+; SI-NEXT: v_writelane_b32 v19, s5, 3
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v19, s4, 4
+; SI-NEXT: v_writelane_b32 v19, s5, 5
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_writelane_b32 v19, s4, 6
+; SI-NEXT: v_writelane_b32 v19, s5, 7
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: s_and_b64 vcc, vcc, exec
+; SI-NEXT: s_cselect_b32 vcc_lo, 1, 0
+; SI-NEXT: s_cmp_lg_u32 vcc_lo, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v2, s44
; SI-NEXT: v_cvt_f32_f16_e32 v1, s46
; SI-NEXT: v_cvt_f32_f16_e32 v3, s99
@@ -78388,67 +79533,8 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_bfe_u32 v10, v3, 8, 8
; SI-NEXT: v_bfe_u32 v8, v2, 8, 8
; SI-NEXT: v_bfe_u32 v7, v1, 8, 8
-; SI-NEXT: s_branch .LBB105_5
-; SI-NEXT: .LBB105_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v19, s4, 0
-; SI-NEXT: v_writelane_b32 v19, s5, 1
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr57
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr47
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: v_writelane_b32 v19, s4, 2
-; SI-NEXT: v_writelane_b32 v19, s5, 3
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v19, s4, 4
-; SI-NEXT: v_writelane_b32 v19, s5, 5
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v19, s4, 6
-; SI-NEXT: v_writelane_b32 v19, s5, 7
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB105_2
-; SI-NEXT: .LBB105_4:
+; SI-NEXT: s_branch .LBB105_6
+; SI-NEXT: .LBB105_5:
; SI-NEXT: v_readlane_b32 s18, v19, 4
; SI-NEXT: v_readlane_b32 s20, v19, 2
; SI-NEXT: v_readlane_b32 s22, v19, 6
@@ -78473,7 +79559,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_readlane_b32 s21, v19, 3
; SI-NEXT: v_readlane_b32 s23, v19, 7
; SI-NEXT: v_readlane_b32 s25, v19, 1
-; SI-NEXT: .LBB105_5: ; %end
+; SI-NEXT: .LBB105_6: ; %end
; SI-NEXT: s_and_b32 s16, s42, 0xff
; SI-NEXT: s_lshl_b32 s17, s24, 8
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -78744,55 +79830,53 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_writelane_b32 v63, s55, 13
; VI-NEXT: v_writelane_b32 v63, s64, 14
; VI-NEXT: v_writelane_b32 v63, s65, 15
-; VI-NEXT: v_writelane_b32 v63, s66, 16
-; VI-NEXT: v_writelane_b32 v63, s67, 17
-; VI-NEXT: v_writelane_b32 v63, s30, 18
-; VI-NEXT: v_writelane_b32 v63, s31, 19
+; VI-NEXT: v_writelane_b32 v63, s30, 16
+; VI-NEXT: v_writelane_b32 v63, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB105_3
+; VI-NEXT: s_cbranch_scc0 .LBB105_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s88, s5, 24
-; VI-NEXT: s_lshr_b32 s36, s5, 16
+; VI-NEXT: s_lshr_b32 s34, s5, 16
; VI-NEXT: s_lshr_b32 s63, s5, 8
-; VI-NEXT: s_lshr_b32 s37, s4, 16
+; VI-NEXT: s_lshr_b32 s35, s4, 16
; VI-NEXT: s_lshr_b32 s79, s4, 8
; VI-NEXT: s_lshr_b32 s89, s29, 24
-; VI-NEXT: s_lshr_b32 s38, s29, 16
+; VI-NEXT: s_lshr_b32 s36, s29, 16
; VI-NEXT: s_lshr_b32 s62, s29, 8
-; VI-NEXT: s_lshr_b32 s39, s28, 16
+; VI-NEXT: s_lshr_b32 s37, s28, 16
; VI-NEXT: s_lshr_b32 s78, s28, 8
; VI-NEXT: s_lshr_b32 s90, s27, 24
-; VI-NEXT: s_lshr_b32 s48, s27, 16
+; VI-NEXT: s_lshr_b32 s38, s27, 16
; VI-NEXT: s_lshr_b32 s61, s27, 8
-; VI-NEXT: s_lshr_b32 s49, s26, 16
+; VI-NEXT: s_lshr_b32 s39, s26, 16
; VI-NEXT: s_lshr_b32 s77, s26, 8
; VI-NEXT: s_lshr_b32 s91, s25, 24
-; VI-NEXT: s_lshr_b32 s50, s25, 16
+; VI-NEXT: s_lshr_b32 s48, s25, 16
; VI-NEXT: s_lshr_b32 s60, s25, 8
-; VI-NEXT: s_lshr_b32 s51, s24, 16
+; VI-NEXT: s_lshr_b32 s49, s24, 16
; VI-NEXT: s_lshr_b32 s76, s24, 8
-; VI-NEXT: s_lshr_b32 s30, s23, 24
-; VI-NEXT: s_lshr_b32 s52, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 24
+; VI-NEXT: s_lshr_b32 s50, s23, 16
; VI-NEXT: s_lshr_b32 s59, s23, 8
-; VI-NEXT: s_lshr_b32 s53, s22, 16
+; VI-NEXT: s_lshr_b32 s51, s22, 16
; VI-NEXT: s_lshr_b32 s75, s22, 8
-; VI-NEXT: s_lshr_b32 s31, s21, 24
-; VI-NEXT: s_lshr_b32 s54, s21, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; VI-NEXT: s_lshr_b32 s52, s21, 16
; VI-NEXT: s_lshr_b32 s58, s21, 8
-; VI-NEXT: s_lshr_b32 s55, s20, 16
+; VI-NEXT: s_lshr_b32 s53, s20, 16
; VI-NEXT: s_lshr_b32 s74, s20, 8
-; VI-NEXT: s_lshr_b32 s34, s19, 24
-; VI-NEXT: s_lshr_b32 s64, s19, 16
+; VI-NEXT: s_lshr_b32 s30, s19, 24
+; VI-NEXT: s_lshr_b32 s54, s19, 16
; VI-NEXT: s_lshr_b32 s57, s19, 8
-; VI-NEXT: s_lshr_b32 s65, s18, 16
+; VI-NEXT: s_lshr_b32 s55, s18, 16
; VI-NEXT: s_lshr_b32 s73, s18, 8
-; VI-NEXT: s_lshr_b32 s35, s17, 24
-; VI-NEXT: s_lshr_b32 s66, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s17, 24
+; VI-NEXT: s_lshr_b32 s64, s17, 16
; VI-NEXT: s_lshr_b32 s56, s17, 8
-; VI-NEXT: s_lshr_b32 s67, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s16, 16
; VI-NEXT: s_lshr_b32 s72, s16, 8
; VI-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; VI-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
@@ -78802,8 +79886,64 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB105_4
-; VI-NEXT: .LBB105_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB105_3
+; VI-NEXT: .LBB105_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: .LBB105_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB105_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s6, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_add_f16_e32 v11, s6, v1
@@ -78921,78 +80061,28 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_bfe_u32 v56, v13, 8, 8
; VI-NEXT: v_bfe_u32 v59, v12, 8, 8
; VI-NEXT: v_bfe_u32 v62, v11, 8, 8
-; VI-NEXT: s_branch .LBB105_5
-; VI-NEXT: .LBB105_3:
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: s_branch .LBB105_2
-; VI-NEXT: .LBB105_4:
+; VI-NEXT: s_branch .LBB105_6
+; VI-NEXT: .LBB105_5:
; VI-NEXT: v_mov_b32_e32 v2, s44
; VI-NEXT: v_mov_b32_e32 v1, s79
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v18, s67
-; VI-NEXT: v_mov_b32_e32 v11, s66
-; VI-NEXT: v_mov_b32_e32 v20, s65
-; VI-NEXT: v_mov_b32_e32 v12, s64
-; VI-NEXT: v_mov_b32_e32 v21, s55
-; VI-NEXT: v_mov_b32_e32 v13, s54
-; VI-NEXT: v_mov_b32_e32 v22, s53
-; VI-NEXT: v_mov_b32_e32 v14, s52
-; VI-NEXT: v_mov_b32_e32 v23, s51
-; VI-NEXT: v_mov_b32_e32 v15, s50
-; VI-NEXT: v_mov_b32_e32 v24, s49
-; VI-NEXT: v_mov_b32_e32 v16, s48
-; VI-NEXT: v_mov_b32_e32 v26, s39
-; VI-NEXT: v_mov_b32_e32 v17, s38
-; VI-NEXT: v_mov_b32_e32 v29, s37
-; VI-NEXT: v_mov_b32_e32 v19, s36
+; VI-NEXT: v_mov_b32_e32 v18, s65
+; VI-NEXT: v_mov_b32_e32 v11, s64
+; VI-NEXT: v_mov_b32_e32 v20, s55
+; VI-NEXT: v_mov_b32_e32 v12, s54
+; VI-NEXT: v_mov_b32_e32 v21, s53
+; VI-NEXT: v_mov_b32_e32 v13, s52
+; VI-NEXT: v_mov_b32_e32 v22, s51
+; VI-NEXT: v_mov_b32_e32 v14, s50
+; VI-NEXT: v_mov_b32_e32 v23, s49
+; VI-NEXT: v_mov_b32_e32 v15, s48
+; VI-NEXT: v_mov_b32_e32 v24, s39
+; VI-NEXT: v_mov_b32_e32 v16, s38
+; VI-NEXT: v_mov_b32_e32 v26, s37
+; VI-NEXT: v_mov_b32_e32 v17, s36
+; VI-NEXT: v_mov_b32_e32 v29, s35
+; VI-NEXT: v_mov_b32_e32 v19, s34
; VI-NEXT: v_mov_b32_e32 v33, s16
; VI-NEXT: v_mov_b32_e32 v25, s17
; VI-NEXT: v_mov_b32_e32 v35, s18
@@ -79013,10 +80103,10 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s89
; VI-NEXT: v_mov_b32_e32 v41, s90
; VI-NEXT: v_mov_b32_e32 v43, s91
-; VI-NEXT: v_mov_b32_e32 v46, s30
-; VI-NEXT: v_mov_b32_e32 v56, s31
-; VI-NEXT: v_mov_b32_e32 v59, s34
-; VI-NEXT: v_mov_b32_e32 v62, s35
+; VI-NEXT: v_mov_b32_e32 v46, vcc_lo
+; VI-NEXT: v_mov_b32_e32 v56, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v59, s30
+; VI-NEXT: v_mov_b32_e32 v62, s31
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v52, s63
; VI-NEXT: v_mov_b32_e32 v54, s78
@@ -79040,7 +80130,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v6, s10
; VI-NEXT: v_mov_b32_e32 v7, s8
; VI-NEXT: v_mov_b32_e32 v8, s6
-; VI-NEXT: .LBB105_5: ; %end
+; VI-NEXT: .LBB105_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v8, v18, v8, s4
; VI-NEXT: v_perm_b32 v1, v33, v1, s4
@@ -79128,10 +80218,8 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: v_readlane_b32 s30, v63, 18
-; VI-NEXT: v_readlane_b32 s31, v63, 19
-; VI-NEXT: v_readlane_b32 s67, v63, 17
-; VI-NEXT: v_readlane_b32 s66, v63, 16
+; VI-NEXT: v_readlane_b32 s30, v63, 16
+; VI-NEXT: v_readlane_b32 s31, v63, 17
; VI-NEXT: v_readlane_b32 s65, v63, 15
; VI-NEXT: v_readlane_b32 s64, v63, 14
; VI-NEXT: v_readlane_b32 s55, v63, 13
@@ -79215,15 +80303,13 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_writelane_b32 v63, s51, 9
; GFX9-NEXT: v_writelane_b32 v63, s52, 10
; GFX9-NEXT: v_writelane_b32 v63, s53, 11
-; GFX9-NEXT: v_writelane_b32 v63, s54, 12
-; GFX9-NEXT: v_writelane_b32 v63, s55, 13
-; GFX9-NEXT: v_writelane_b32 v63, s30, 14
-; GFX9-NEXT: v_writelane_b32 v63, s31, 15
+; GFX9-NEXT: v_writelane_b32 v63, s30, 12
+; GFX9-NEXT: v_writelane_b32 v63, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB105_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s56, s5, 24
; GFX9-NEXT: s_lshr_b32 s57, s5, 16
@@ -79249,22 +80335,22 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s93, s23, 16
; GFX9-NEXT: s_lshr_b32 s95, s23, 8
; GFX9-NEXT: s_lshr_b32 s94, s22, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 8
-; GFX9-NEXT: s_lshr_b32 s31, s21, 24
-; GFX9-NEXT: s_lshr_b32 s34, s21, 16
-; GFX9-NEXT: s_lshr_b32 s36, s21, 8
-; GFX9-NEXT: s_lshr_b32 s35, s20, 16
-; GFX9-NEXT: s_lshr_b32 s37, s20, 8
-; GFX9-NEXT: s_lshr_b32 s38, s19, 24
-; GFX9-NEXT: s_lshr_b32 s39, s19, 16
-; GFX9-NEXT: s_lshr_b32 s49, s19, 8
-; GFX9-NEXT: s_lshr_b32 s48, s18, 16
-; GFX9-NEXT: s_lshr_b32 s50, s18, 8
-; GFX9-NEXT: s_lshr_b32 s51, s17, 24
-; GFX9-NEXT: s_lshr_b32 s52, s17, 16
-; GFX9-NEXT: s_lshr_b32 s54, s17, 8
-; GFX9-NEXT: s_lshr_b32 s53, s16, 16
-; GFX9-NEXT: s_lshr_b32 s55, s16, 8
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 8
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s21, 24
+; GFX9-NEXT: s_lshr_b32 s30, s21, 16
+; GFX9-NEXT: s_lshr_b32 s34, s21, 8
+; GFX9-NEXT: s_lshr_b32 s31, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 8
+; GFX9-NEXT: s_lshr_b32 s36, s19, 24
+; GFX9-NEXT: s_lshr_b32 s37, s19, 16
+; GFX9-NEXT: s_lshr_b32 s39, s19, 8
+; GFX9-NEXT: s_lshr_b32 s38, s18, 16
+; GFX9-NEXT: s_lshr_b32 s48, s18, 8
+; GFX9-NEXT: s_lshr_b32 s49, s17, 24
+; GFX9-NEXT: s_lshr_b32 s50, s17, 16
+; GFX9-NEXT: s_lshr_b32 s52, s17, 8
+; GFX9-NEXT: s_lshr_b32 s51, s16, 16
+; GFX9-NEXT: s_lshr_b32 s53, s16, 8
; GFX9-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
; GFX9-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
@@ -79273,8 +80359,64 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB105_4
-; GFX9-NEXT: .LBB105_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB105_3
+; GFX9-NEXT: .LBB105_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: .LBB105_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v1, 0x200
; GFX9-NEXT: v_pk_add_f16 v19, s17, v1 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s16, v1 op_sel_hi:[1,0]
@@ -79346,58 +80488,8 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v18
-; GFX9-NEXT: s_branch .LBB105_5
-; GFX9-NEXT: .LBB105_3:
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: s_branch .LBB105_2
-; GFX9-NEXT: .LBB105_4:
+; GFX9-NEXT: s_branch .LBB105_6
+; GFX9-NEXT: .LBB105_5:
; GFX9-NEXT: v_mov_b32_e32 v20, s44
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -79419,22 +80511,22 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v4, s29
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NEXT: v_mov_b32_e32 v15, s55
-; GFX9-NEXT: v_mov_b32_e32 v16, s53
-; GFX9-NEXT: v_mov_b32_e32 v62, s54
-; GFX9-NEXT: v_mov_b32_e32 v59, s52
-; GFX9-NEXT: v_mov_b32_e32 v60, s51
-; GFX9-NEXT: v_mov_b32_e32 v58, s50
-; GFX9-NEXT: v_mov_b32_e32 v61, s48
-; GFX9-NEXT: v_mov_b32_e32 v57, s49
-; GFX9-NEXT: v_mov_b32_e32 v47, s39
-; GFX9-NEXT: v_mov_b32_e32 v56, s38
-; GFX9-NEXT: v_mov_b32_e32 v46, s37
-; GFX9-NEXT: v_mov_b32_e32 v45, s35
-; GFX9-NEXT: v_mov_b32_e32 v44, s36
-; GFX9-NEXT: v_mov_b32_e32 v42, s34
-; GFX9-NEXT: v_mov_b32_e32 v43, s31
-; GFX9-NEXT: v_mov_b32_e32 v41, s30
+; GFX9-NEXT: v_mov_b32_e32 v15, s53
+; GFX9-NEXT: v_mov_b32_e32 v16, s51
+; GFX9-NEXT: v_mov_b32_e32 v62, s52
+; GFX9-NEXT: v_mov_b32_e32 v59, s50
+; GFX9-NEXT: v_mov_b32_e32 v60, s49
+; GFX9-NEXT: v_mov_b32_e32 v58, s48
+; GFX9-NEXT: v_mov_b32_e32 v61, s38
+; GFX9-NEXT: v_mov_b32_e32 v57, s39
+; GFX9-NEXT: v_mov_b32_e32 v47, s37
+; GFX9-NEXT: v_mov_b32_e32 v56, s36
+; GFX9-NEXT: v_mov_b32_e32 v46, s35
+; GFX9-NEXT: v_mov_b32_e32 v45, s31
+; GFX9-NEXT: v_mov_b32_e32 v44, s34
+; GFX9-NEXT: v_mov_b32_e32 v42, s30
+; GFX9-NEXT: v_mov_b32_e32 v43, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v41, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v40, s94
; GFX9-NEXT: v_mov_b32_e32 v55, s95
; GFX9-NEXT: v_mov_b32_e32 v53, s93
@@ -79468,7 +80560,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v20, s40
; GFX9-NEXT: v_mov_b32_e32 v21, s14
-; GFX9-NEXT: .LBB105_5: ; %end
+; GFX9-NEXT: .LBB105_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v16, v16, v25, s4
; GFX9-NEXT: v_perm_b32 v15, v18, v15, s4
@@ -79535,10 +80627,8 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
-; GFX9-NEXT: v_readlane_b32 s30, v63, 14
-; GFX9-NEXT: v_readlane_b32 s31, v63, 15
-; GFX9-NEXT: v_readlane_b32 s55, v63, 13
-; GFX9-NEXT: v_readlane_b32 s54, v63, 12
+; GFX9-NEXT: v_readlane_b32 s30, v63, 12
+; GFX9-NEXT: v_readlane_b32 s31, v63, 13
; GFX9-NEXT: v_readlane_b32 s53, v63, 11
; GFX9-NEXT: v_readlane_b32 s52, v63, 10
; GFX9-NEXT: v_readlane_b32 s51, v63, 9
@@ -79608,12 +80698,11 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-NEXT: s_mov_b32 s42, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s43, s27, 24
; GFX11-NEXT: s_lshr_b32 s44, s27, 16
@@ -79644,17 +80733,17 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s93, s17, 16
; GFX11-NEXT: s_lshr_b32 s95, s17, 8
; GFX11-NEXT: s_lshr_b32 s94, s16, 16
-; GFX11-NEXT: s_lshr_b32 vcc_hi, s16, 8
-; GFX11-NEXT: s_lshr_b32 s30, s3, 24
-; GFX11-NEXT: s_lshr_b32 s31, s3, 16
-; GFX11-NEXT: s_lshr_b32 s35, s3, 8
-; GFX11-NEXT: s_lshr_b32 s34, s2, 16
-; GFX11-NEXT: s_lshr_b32 s36, s2, 8
-; GFX11-NEXT: s_lshr_b32 s37, s1, 24
-; GFX11-NEXT: s_lshr_b32 s38, s1, 16
-; GFX11-NEXT: s_lshr_b32 s48, s1, 8
-; GFX11-NEXT: s_lshr_b32 s39, s0, 16
-; GFX11-NEXT: s_lshr_b32 s49, s0, 8
+; GFX11-NEXT: s_lshr_b32 vcc_lo, s16, 8
+; GFX11-NEXT: s_lshr_b32 vcc_hi, s3, 24
+; GFX11-NEXT: s_lshr_b32 s30, s3, 16
+; GFX11-NEXT: s_lshr_b32 s34, s3, 8
+; GFX11-NEXT: s_lshr_b32 s31, s2, 16
+; GFX11-NEXT: s_lshr_b32 s35, s2, 8
+; GFX11-NEXT: s_lshr_b32 s36, s1, 24
+; GFX11-NEXT: s_lshr_b32 s37, s1, 16
+; GFX11-NEXT: s_lshr_b32 s39, s1, 8
+; GFX11-NEXT: s_lshr_b32 s38, s0, 16
+; GFX11-NEXT: s_lshr_b32 s48, s0, 8
; GFX11-NEXT: s_lshr_b64 s[40:41], s[26:27], 24
; GFX11-NEXT: s_lshr_b64 s[28:29], s[24:25], 24
; GFX11-NEXT: s_lshr_b64 s[14:15], s[22:23], 24
@@ -79663,9 +80752,64 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB105_3
+; GFX11-NEXT: .LBB105_2:
+; GFX11-NEXT: s_mov_b32 s42, -1
+; GFX11-NEXT: ; implicit-def: $sgpr48
+; GFX11-NEXT: ; implicit-def: $sgpr38
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr39
+; GFX11-NEXT: ; implicit-def: $sgpr37
+; GFX11-NEXT: ; implicit-def: $sgpr36
+; GFX11-NEXT: ; implicit-def: $sgpr35
+; GFX11-NEXT: ; implicit-def: $sgpr31
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr34
+; GFX11-NEXT: ; implicit-def: $sgpr30
+; GFX11-NEXT: ; implicit-def: $vcc_hi
+; GFX11-NEXT: ; implicit-def: $vcc_lo
+; GFX11-NEXT: ; implicit-def: $sgpr94
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr95
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr28
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: .LBB105_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v4, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v3, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v6, 0x200, s23 op_sel_hi:[0,1]
@@ -79730,58 +80874,8 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v83, 8, v18
; GFX11-NEXT: v_lshrrev_b32_e32 v96, 16, v17
; GFX11-NEXT: v_lshrrev_b32_e32 v85, 8, v17
-; GFX11-NEXT: s_branch .LBB105_5
-; GFX11-NEXT: .LBB105_3:
-; GFX11-NEXT: ; implicit-def: $sgpr49
-; GFX11-NEXT: ; implicit-def: $sgpr39
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr48
-; GFX11-NEXT: ; implicit-def: $sgpr38
-; GFX11-NEXT: ; implicit-def: $sgpr37
-; GFX11-NEXT: ; implicit-def: $sgpr36
-; GFX11-NEXT: ; implicit-def: $sgpr34
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr35
-; GFX11-NEXT: ; implicit-def: $sgpr31
-; GFX11-NEXT: ; implicit-def: $sgpr30
-; GFX11-NEXT: ; implicit-def: $vcc_hi
-; GFX11-NEXT: ; implicit-def: $sgpr94
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr95
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr28
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: s_branch .LBB105_2
-; GFX11-NEXT: .LBB105_4:
+; GFX11-NEXT: s_branch .LBB105_6
+; GFX11-NEXT: .LBB105_5:
; GFX11-NEXT: v_dual_mov_b32 v17, s0 :: v_dual_mov_b32 v18, s1
; GFX11-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s3
; GFX11-NEXT: v_dual_mov_b32 v11, s16 :: v_dual_mov_b32 v12, s17
@@ -79790,12 +80884,12 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v5, s22 :: v_dual_mov_b32 v6, s23
; GFX11-NEXT: v_dual_mov_b32 v3, s24 :: v_dual_mov_b32 v4, s25
; GFX11-NEXT: v_dual_mov_b32 v1, s26 :: v_dual_mov_b32 v2, s27
-; GFX11-NEXT: v_dual_mov_b32 v85, s49 :: v_dual_mov_b32 v96, s39
-; GFX11-NEXT: v_dual_mov_b32 v83, s48 :: v_dual_mov_b32 v86, s38
-; GFX11-NEXT: v_dual_mov_b32 v87, s37 :: v_dual_mov_b32 v82, s36
-; GFX11-NEXT: v_dual_mov_b32 v84, s34 :: v_dual_mov_b32 v71, s35
-; GFX11-NEXT: v_dual_mov_b32 v80, s31 :: v_dual_mov_b32 v81, s30
-; GFX11-NEXT: v_dual_mov_b32 v67, vcc_hi :: v_dual_mov_b32 v70, s94
+; GFX11-NEXT: v_dual_mov_b32 v85, s48 :: v_dual_mov_b32 v96, s38
+; GFX11-NEXT: v_dual_mov_b32 v83, s39 :: v_dual_mov_b32 v86, s37
+; GFX11-NEXT: v_dual_mov_b32 v87, s36 :: v_dual_mov_b32 v82, s35
+; GFX11-NEXT: v_dual_mov_b32 v84, s31 :: v_dual_mov_b32 v71, s34
+; GFX11-NEXT: v_dual_mov_b32 v80, s30 :: v_dual_mov_b32 v81, vcc_hi
+; GFX11-NEXT: v_dual_mov_b32 v67, vcc_lo :: v_dual_mov_b32 v70, s94
; GFX11-NEXT: v_dual_mov_b32 v65, s95 :: v_dual_mov_b32 v68, s93
; GFX11-NEXT: v_dual_mov_b32 v69, s92 :: v_dual_mov_b32 v64, s91
; GFX11-NEXT: v_dual_mov_b32 v66, s89 :: v_dual_mov_b32 v53, s90
@@ -79814,7 +80908,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v20, s14 :: v_dual_mov_b32 v21, s12
; GFX11-NEXT: v_dual_mov_b32 v22, s10 :: v_dual_mov_b32 v23, s8
; GFX11-NEXT: v_dual_mov_b32 v24, s6 :: v_dual_mov_b32 v25, s4
-; GFX11-NEXT: .LBB105_5: ; %end
+; GFX11-NEXT: .LBB105_6: ; %end
; GFX11-NEXT: v_perm_b32 v86, v86, v87, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_perm_b32 v24, v84, v24, 0xc0c0004
@@ -79880,14 +80974,13 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v2, v4, v17
; GFX11-NEXT: v_or_b32_e32 v3, v19, v15
; GFX11-NEXT: v_or_b32_e32 v4, v16, v18
-; GFX11-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: scratch_store_b128 v0, v[80:83], off
; GFX11-NEXT: scratch_store_b128 v0, v[11:14], off offset:16
; GFX11-NEXT: scratch_store_b128 v0, v[7:10], off offset:32
; GFX11-NEXT: scratch_store_b128 v0, v[1:4], off offset:48
-; GFX11-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-NEXT: v_readlane_b32 s38, v40, 4
@@ -82468,276 +83561,274 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr44 : SGPR spill to VGPR lane
-; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v44, s23, 0
-; SI-NEXT: v_writelane_b32 v44, s22, 1
-; SI-NEXT: v_writelane_b32 v44, s21, 2
-; SI-NEXT: v_writelane_b32 v44, s20, 3
-; SI-NEXT: v_writelane_b32 v44, s4, 4
-; SI-NEXT: s_mov_b32 s82, s29
-; SI-NEXT: s_mov_b32 s85, s25
-; SI-NEXT: s_mov_b32 s29, s24
-; SI-NEXT: s_mov_b32 s44, s16
-; SI-NEXT: v_readfirstlane_b32 s64, v30
+; SI-NEXT: s_mov_b32 s96, s27
+; SI-NEXT: v_writelane_b32 v44, s28, 0
+; SI-NEXT: v_writelane_b32 v44, s26, 1
+; SI-NEXT: v_writelane_b32 v44, s23, 2
+; SI-NEXT: v_writelane_b32 v44, s22, 3
+; SI-NEXT: v_writelane_b32 v44, s16, 4
+; SI-NEXT: s_mov_b32 s27, s25
+; SI-NEXT: s_mov_b32 s85, s24
+; SI-NEXT: v_readfirstlane_b32 s69, v30
; SI-NEXT: v_readfirstlane_b32 s55, v29
-; SI-NEXT: v_readfirstlane_b32 s66, v28
-; SI-NEXT: v_readfirstlane_b32 s69, v27
-; SI-NEXT: v_readfirstlane_b32 s68, v26
-; SI-NEXT: v_readfirstlane_b32 s39, v25
-; SI-NEXT: v_readfirstlane_b32 s50, v24
-; SI-NEXT: v_readfirstlane_b32 s65, v23
-; SI-NEXT: v_readfirstlane_b32 s51, v22
-; SI-NEXT: v_readfirstlane_b32 s37, v21
-; SI-NEXT: v_readfirstlane_b32 s53, v20
-; SI-NEXT: v_readfirstlane_b32 s88, v19
-; SI-NEXT: v_readfirstlane_b32 s31, v18
-; SI-NEXT: v_readfirstlane_b32 s95, v17
-; SI-NEXT: v_readfirstlane_b32 s48, v15
-; SI-NEXT: v_readfirstlane_b32 s35, v14
+; SI-NEXT: v_readfirstlane_b32 s65, v28
+; SI-NEXT: v_readfirstlane_b32 s71, v27
+; SI-NEXT: v_readfirstlane_b32 s67, v26
+; SI-NEXT: v_readfirstlane_b32 s38, v25
+; SI-NEXT: v_readfirstlane_b32 s68, v24
+; SI-NEXT: v_readfirstlane_b32 s49, v23
+; SI-NEXT: v_readfirstlane_b32 s94, v22
+; SI-NEXT: v_readfirstlane_b32 s50, v21
+; SI-NEXT: v_readfirstlane_b32 s88, v20
+; SI-NEXT: v_readfirstlane_b32 s66, v19
+; SI-NEXT: v_readfirstlane_b32 s54, v18
+; SI-NEXT: v_readfirstlane_b32 s86, v17
+; SI-NEXT: v_readfirstlane_b32 s31, v16
+; SI-NEXT: v_readfirstlane_b32 s37, v15
+; SI-NEXT: v_readfirstlane_b32 s39, v14
; SI-NEXT: v_readfirstlane_b32 s34, v13
-; SI-NEXT: v_readfirstlane_b32 s52, v12
-; SI-NEXT: v_readfirstlane_b32 s16, v11
-; SI-NEXT: v_readfirstlane_b32 s96, v10
-; SI-NEXT: v_readfirstlane_b32 s20, v9
-; SI-NEXT: v_readfirstlane_b32 s89, v8
-; SI-NEXT: v_readfirstlane_b32 s92, v7
+; SI-NEXT: v_readfirstlane_b32 s36, v12
+; SI-NEXT: v_readfirstlane_b32 s53, v11
+; SI-NEXT: v_readfirstlane_b32 s64, v10
+; SI-NEXT: v_readfirstlane_b32 s89, v9
+; SI-NEXT: v_readfirstlane_b32 s90, v8
+; SI-NEXT: v_readfirstlane_b32 s16, v7
; SI-NEXT: v_readfirstlane_b32 s91, v6
-; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s38, v3
-; SI-NEXT: v_readfirstlane_b32 s49, v2
-; SI-NEXT: v_readfirstlane_b32 s54, v1
-; SI-NEXT: v_readfirstlane_b32 s90, v0
+; SI-NEXT: v_readfirstlane_b32 s28, v5
+; SI-NEXT: v_readfirstlane_b32 s93, v4
+; SI-NEXT: v_readfirstlane_b32 s35, v3
+; SI-NEXT: v_readfirstlane_b32 s51, v2
+; SI-NEXT: v_readfirstlane_b32 s23, v1
+; SI-NEXT: v_readfirstlane_b32 s79, v0
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; SI-NEXT: v_readfirstlane_b32 s58, v32
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 s56, v32
-; SI-NEXT: v_readfirstlane_b32 s86, v33
-; SI-NEXT: v_readfirstlane_b32 s58, v34
-; SI-NEXT: v_readfirstlane_b32 s99, v35
-; SI-NEXT: v_readfirstlane_b32 s24, v36
+; SI-NEXT: v_readfirstlane_b32 s84, v33
+; SI-NEXT: v_readfirstlane_b32 s47, v34
+; SI-NEXT: v_readfirstlane_b32 s30, v35
+; SI-NEXT: v_readfirstlane_b32 s25, v36
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_readfirstlane_b32 s78, v37
+; SI-NEXT: v_readfirstlane_b32 s98, v37
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s25, v38
+; SI-NEXT: v_readfirstlane_b32 s26, v38
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s21, v39
+; SI-NEXT: v_readfirstlane_b32 s22, v39
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s22, v48
+; SI-NEXT: v_readfirstlane_b32 s99, v48
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s23, v49
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s87, v50
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s47, v52
-; SI-NEXT: v_readfirstlane_b32 s83, v51
+; SI-NEXT: v_readfirstlane_b32 s70, v49
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_readfirstlane_b32 s56, v51
+; SI-NEXT: v_writelane_b32 v44, s56, 5
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s84, v53
+; SI-NEXT: v_readfirstlane_b32 s57, v53
+; SI-NEXT: v_writelane_b32 v44, s57, 6
+; SI-NEXT: v_readfirstlane_b32 s59, v52
+; SI-NEXT: v_writelane_b32 v44, s58, 7
+; SI-NEXT: v_readfirstlane_b32 s78, v50
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s36, v54
+; SI-NEXT: v_readfirstlane_b32 s44, v54
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s79, v55
+; SI-NEXT: v_readfirstlane_b32 s87, v55
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s71, v40
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v40
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s98, v41
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v41
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s81, v42
-; SI-NEXT: v_writelane_b32 v44, s47, 5
-; SI-NEXT: v_writelane_b32 v44, s58, 6
+; SI-NEXT: v_readfirstlane_b32 s83, v42
+; SI-NEXT: v_writelane_b32 v44, s59, 8
+; SI-NEXT: v_writelane_b32 v44, s47, 9
; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_and_b32 s4, s44, 0xff
+; SI-NEXT: v_readlane_b32 s4, v44, 4
+; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
; SI-NEXT: s_or_b32 s12, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s19, 24
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s29, 0xff
-; SI-NEXT: s_lshl_b32 s6, s85, 8
+; SI-NEXT: s_and_b32 s5, s85, 0xff
+; SI-NEXT: s_lshl_b32 s6, s27, 8
; SI-NEXT: s_or_b32 s13, s5, s6
-; SI-NEXT: s_and_b32 s5, s26, 0xff
+; SI-NEXT: v_readlane_b32 s5, v44, 1
+; SI-NEXT: s_and_b32 s5, s5, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s6, s27, 24
+; SI-NEXT: s_lshl_b32 s6, s96, 24
; SI-NEXT: s_or_b32 s6, s6, s5
-; SI-NEXT: s_and_b32 s5, s49, 0xff
-; SI-NEXT: s_lshl_b32 s7, s38, 8
+; SI-NEXT: s_and_b32 s5, s51, 0xff
+; SI-NEXT: s_lshl_b32 s7, s35, 8
; SI-NEXT: s_or_b32 s14, s5, s7
-; SI-NEXT: s_and_b32 s5, s30, 0xff
+; SI-NEXT: s_and_b32 s5, s93, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s93, 24
+; SI-NEXT: s_lshl_b32 s7, s28, 24
; SI-NEXT: s_or_b32 s8, s7, s5
-; SI-NEXT: s_and_b32 s5, s96, 0xff
-; SI-NEXT: s_lshl_b32 s7, s16, 8
+; SI-NEXT: s_and_b32 s5, s64, 0xff
+; SI-NEXT: s_lshl_b32 s7, s53, 8
; SI-NEXT: s_or_b32 s15, s5, s7
-; SI-NEXT: s_and_b32 s5, s52, 0xff
+; SI-NEXT: s_and_b32 s5, s36, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s34, 24
; SI-NEXT: s_or_b32 s10, s7, s5
-; SI-NEXT: s_and_b32 s5, s31, 0xff
-; SI-NEXT: s_lshl_b32 s7, s88, 8
+; SI-NEXT: s_and_b32 s5, s54, 0xff
+; SI-NEXT: s_lshl_b32 s7, s66, 8
; SI-NEXT: s_or_b32 s40, s5, s7
-; SI-NEXT: s_and_b32 s5, s53, 0xff
+; SI-NEXT: s_and_b32 s5, s88, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s37, 24
+; SI-NEXT: s_lshl_b32 s7, s50, 24
; SI-NEXT: s_or_b32 s60, s7, s5
-; SI-NEXT: s_and_b32 s5, s68, 0xff
-; SI-NEXT: s_lshl_b32 s7, s69, 8
+; SI-NEXT: s_and_b32 s5, s67, 0xff
+; SI-NEXT: s_lshl_b32 s7, s71, 8
; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s66, 0xff
+; SI-NEXT: s_and_b32 s5, s65, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s55, 24
; SI-NEXT: s_or_b32 s62, s7, s5
-; SI-NEXT: s_and_b32 s5, s79, 0xff
-; SI-NEXT: s_lshl_b32 s7, s36, 8
+; SI-NEXT: s_and_b32 s5, s87, 0xff
+; SI-NEXT: s_lshl_b32 s7, s44, 8
; SI-NEXT: s_or_b32 s42, s5, s7
-; SI-NEXT: s_and_b32 s5, s21, 0xff
-; SI-NEXT: s_lshl_b32 s7, s25, 8
+; SI-NEXT: s_and_b32 s5, s22, 0xff
+; SI-NEXT: s_lshl_b32 s7, s26, 8
; SI-NEXT: s_or_b32 s43, s5, s7
-; SI-NEXT: v_readlane_b32 s5, v44, 3
-; SI-NEXT: v_readlane_b32 s7, v44, 2
-; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 8
+; SI-NEXT: s_and_b32 s5, s20, 0xff
+; SI-NEXT: s_lshl_b32 s7, s21, 8
; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readlane_b32 s7, v44, 1
+; SI-NEXT: v_readlane_b32 s7, v44, 3
; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: v_readlane_b32 s9, v44, 0
+; SI-NEXT: v_readlane_b32 s9, v44, 2
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_lshl_b32 s9, s9, 24
-; SI-NEXT: s_or_b32 s57, s9, s7
-; SI-NEXT: s_and_b32 s7, s28, 0xff
-; SI-NEXT: s_lshl_b32 s9, s82, 8
+; SI-NEXT: s_or_b32 s76, s9, s7
+; SI-NEXT: v_readlane_b32 s7, v44, 0
+; SI-NEXT: s_and_b32 s7, s7, 0xff
+; SI-NEXT: s_lshl_b32 s9, s29, 8
; SI-NEXT: s_or_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s90, 0xff
+; SI-NEXT: s_and_b32 s9, s79, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_lshl_b32 s11, s54, 24
+; SI-NEXT: s_lshl_b32 s11, s23, 24
; SI-NEXT: s_or_b32 s77, s11, s9
; SI-NEXT: s_and_b32 s9, s91, 0xff
-; SI-NEXT: s_lshl_b32 s11, s92, 8
+; SI-NEXT: s_lshl_b32 s11, s16, 8
; SI-NEXT: s_or_b32 s9, s9, s11
-; SI-NEXT: s_and_b32 s11, s89, 0xff
-; SI-NEXT: s_mov_b32 s72, s44
+; SI-NEXT: s_and_b32 s11, s90, 0xff
+; SI-NEXT: s_mov_b32 s24, s85
; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_lshl_b32 s44, s20, 24
-; SI-NEXT: s_mov_b32 s59, s16
-; SI-NEXT: s_or_b32 vcc_lo, s44, s11
-; SI-NEXT: s_and_b32 s11, s35, 0xff
-; SI-NEXT: s_lshl_b32 s44, s48, 8
-; SI-NEXT: v_readlane_b32 s16, v44, 4
+; SI-NEXT: s_mov_b32 s85, s26
+; SI-NEXT: s_mov_b32 s26, s44
+; SI-NEXT: s_lshl_b32 s44, s89, 24
+; SI-NEXT: v_writelane_b32 v44, s55, 10
+; SI-NEXT: s_mov_b32 s55, s34
+; SI-NEXT: s_mov_b32 s34, s23
+; SI-NEXT: s_or_b32 s23, s44, s11
+; SI-NEXT: s_and_b32 s11, s39, 0xff
+; SI-NEXT: s_lshl_b32 s44, s37, 8
; SI-NEXT: s_or_b32 s11, s11, s44
-; SI-NEXT: s_and_b32 s44, s16, 0xff
+; SI-NEXT: s_and_b32 s44, s31, 0xff
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s45, s95, 24
-; SI-NEXT: s_or_b32 vcc_hi, s45, s44
-; SI-NEXT: s_and_b32 s44, s51, 0xff
-; SI-NEXT: s_lshl_b32 s45, s65, 8
+; SI-NEXT: s_lshl_b32 s45, s86, 24
+; SI-NEXT: v_writelane_b32 v44, s67, 11
+; SI-NEXT: v_writelane_b32 v44, s50, 12
+; SI-NEXT: s_mov_b32 s50, s21
+; SI-NEXT: s_or_b32 s21, s45, s44
+; SI-NEXT: s_and_b32 s44, s94, 0xff
+; SI-NEXT: s_lshl_b32 s45, s49, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s50, 0xff
+; SI-NEXT: s_and_b32 s45, s68, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s39, 24
+; SI-NEXT: s_lshl_b32 s46, s38, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s67, s20
+; SI-NEXT: s_mov_b32 s95, s20
; SI-NEXT: s_or_b32 s20, s46, s45
; SI-NEXT: s_or_b32 s61, s44, s20
-; SI-NEXT: s_and_b32 s44, s64, 0xff
-; SI-NEXT: s_lshl_b32 s45, s81, 8
+; SI-NEXT: s_and_b32 s44, s69, 0xff
+; SI-NEXT: s_lshl_b32 s45, s83, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s98, 0xff
+; SI-NEXT: s_and_b32 s45, vcc_hi, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s71, 24
+; SI-NEXT: s_lshl_b32 s46, vcc_lo, 24
+; SI-NEXT: s_mov_b32 s52, s35
+; SI-NEXT: s_mov_b32 s35, s24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s70, s39
-; SI-NEXT: s_mov_b32 s39, s91
-; SI-NEXT: s_mov_b32 s91, s82
-; SI-NEXT: s_mov_b32 s82, s88
-; SI-NEXT: s_mov_b32 s88, s18
+; SI-NEXT: s_mov_b32 s24, s69
+; SI-NEXT: s_mov_b32 s69, s53
+; SI-NEXT: s_mov_b32 s53, s31
+; SI-NEXT: s_mov_b32 s31, s96
+; SI-NEXT: s_mov_b32 s96, s18
; SI-NEXT: s_or_b32 s18, s46, s45
; SI-NEXT: s_or_b32 s63, s44, s18
-; SI-NEXT: s_and_b32 s44, s83, 0xff
-; SI-NEXT: s_lshl_b32 s45, s87, 8
+; SI-NEXT: s_and_b32 s44, s56, 0xff
+; SI-NEXT: s_lshl_b32 s45, s78, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s23, 0xff
+; SI-NEXT: s_and_b32 s45, s70, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s22, 24
-; SI-NEXT: s_mov_b32 s80, s31
-; SI-NEXT: s_mov_b32 s31, s53
-; SI-NEXT: s_mov_b32 s53, s38
-; SI-NEXT: s_mov_b32 s38, s90
+; SI-NEXT: s_lshl_b32 s46, s99, 24
+; SI-NEXT: s_mov_b32 s56, s22
+; SI-NEXT: s_mov_b32 s22, s87
+; SI-NEXT: s_mov_b32 s87, s83
+; SI-NEXT: s_mov_b32 s83, s54
+; SI-NEXT: s_mov_b32 s54, s39
+; SI-NEXT: s_mov_b32 s39, s90
; SI-NEXT: s_mov_b32 s90, s19
; SI-NEXT: s_or_b32 s19, s46, s45
-; SI-NEXT: s_and_b32 s45, s84, 0xff
+; SI-NEXT: s_and_b32 s45, s57, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s47, 24
+; SI-NEXT: s_lshl_b32 s46, s59, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_mov_b32 s97, s22
-; SI-NEXT: s_mov_b32 s22, s55
-; SI-NEXT: s_mov_b32 s55, s49
-; SI-NEXT: s_mov_b32 s49, s92
-; SI-NEXT: s_mov_b32 s92, s72
; SI-NEXT: s_or_b32 s72, s46, s45
; SI-NEXT: s_or_b32 s73, s44, s19
-; SI-NEXT: s_and_b32 s44, s99, 0xff
-; SI-NEXT: s_lshl_b32 s45, s58, 8
+; SI-NEXT: s_and_b32 s44, s30, 0xff
+; SI-NEXT: s_lshl_b32 s45, s47, 8
; SI-NEXT: s_or_b32 s44, s44, s45
-; SI-NEXT: s_and_b32 s45, s86, 0xff
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_and_b32 s45, s84, 0xff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s56, 24
-; SI-NEXT: s_or_b32 s5, s5, s57
-; SI-NEXT: s_mov_b32 s76, s56
-; SI-NEXT: s_mov_b32 s56, s86
-; SI-NEXT: s_mov_b32 s86, s81
-; SI-NEXT: s_mov_b32 s81, s65
-; SI-NEXT: s_mov_b32 s65, s48
-; SI-NEXT: s_mov_b32 s48, s89
+; SI-NEXT: s_lshl_b32 s46, s58, 24
+; SI-NEXT: s_mov_b32 s97, s70
+; SI-NEXT: s_mov_b32 s70, s66
+; SI-NEXT: s_mov_b32 s66, s36
+; SI-NEXT: s_mov_b32 s36, s89
; SI-NEXT: s_mov_b32 s89, s17
; SI-NEXT: s_or_b32 s17, s46, s45
-; SI-NEXT: s_and_b32 s44, s44, 0xffff
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: v_writelane_b32 v44, s68, 11
-; SI-NEXT: s_mov_b32 s94, s51
-; SI-NEXT: s_mov_b32 s51, s95
-; SI-NEXT: s_mov_b32 s95, s27
-; SI-NEXT: s_or_b32 s75, s44, s17
-; SI-NEXT: s_mov_b32 s44, s93
-; SI-NEXT: s_mov_b32 s93, s28
-; SI-NEXT: s_and_b32 s28, s42, 0xffff
-; SI-NEXT: s_and_b32 s27, s43, 0xffff
-; SI-NEXT: s_or_b32 s42, s12, s4
-; SI-NEXT: s_mov_b32 s43, s5
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: s_or_b32 s9, s9, vcc_lo
-; SI-NEXT: s_and_b32 s45, s78, 0xff
-; SI-NEXT: v_writelane_b32 v44, s4, 7
+; SI-NEXT: s_and_b32 s45, s98, 0xff
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_lshl_b32 s45, s45, 16
-; SI-NEXT: s_lshl_b32 s46, s24, 24
-; SI-NEXT: v_writelane_b32 v44, s5, 8
-; SI-NEXT: s_lshr_b64 s[4:5], s[8:9], 16
+; SI-NEXT: s_lshl_b32 s46, s25, 24
+; SI-NEXT: s_and_b32 s44, s44, 0xffff
+; SI-NEXT: s_or_b32 s5, s5, s76
; SI-NEXT: s_or_b32 s7, s7, s77
-; SI-NEXT: s_or_b32 s11, s11, vcc_hi
-; SI-NEXT: s_mov_b32 s68, s34
+; SI-NEXT: s_or_b32 s9, s9, s23
+; SI-NEXT: s_mov_b32 s67, s27
+; SI-NEXT: s_or_b32 s11, s11, s21
+; SI-NEXT: s_mov_b32 s82, s65
+; SI-NEXT: s_mov_b32 s65, s38
+; SI-NEXT: s_mov_b32 s38, s91
+; SI-NEXT: s_mov_b32 s91, s29
; SI-NEXT: s_or_b32 s74, s46, s45
+; SI-NEXT: s_or_b32 s75, s44, s17
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_and_b32 s58, s15, 0xffff
+; SI-NEXT: s_mov_b32 s48, s16
; SI-NEXT: s_and_b32 s16, s40, 0xffff
-; SI-NEXT: s_mov_b32 s34, s29
; SI-NEXT: s_and_b32 s29, s41, 0xffff
-; SI-NEXT: v_writelane_b32 v44, s4, 9
-; SI-NEXT: s_mov_b32 s45, s78
-; SI-NEXT: s_mov_b32 s78, s84
-; SI-NEXT: s_mov_b32 s84, s69
-; SI-NEXT: s_mov_b32 s69, s59
+; SI-NEXT: s_mov_b32 s45, s98
+; SI-NEXT: s_mov_b32 s98, s28
+; SI-NEXT: s_and_b32 s28, s42, 0xffff
+; SI-NEXT: s_and_b32 s27, s43, 0xffff
+; SI-NEXT: s_mov_b32 s44, s93
+; SI-NEXT: s_or_b32 s42, s12, s4
+; SI-NEXT: s_mov_b32 s43, s5
+; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
; SI-NEXT: s_or_b32 s40, s13, s6
; SI-NEXT: s_mov_b32 s41, s7
; SI-NEXT: s_lshr_b64 s[46:47], s[6:7], 16
; SI-NEXT: s_or_b32 s14, s14, s8
; SI-NEXT: s_mov_b32 s15, s9
-; SI-NEXT: v_writelane_b32 v44, s5, 10
+; SI-NEXT: s_lshr_b64 s[92:93], s[8:9], 16
; SI-NEXT: s_or_b32 s12, s58, s10
; SI-NEXT: s_mov_b32 s13, s11
; SI-NEXT: s_lshr_b64 s[58:59], s[10:11], 16
@@ -82753,63 +83844,63 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s4, s27, s74
; SI-NEXT: s_mov_b32 s5, s75
; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
-; SI-NEXT: s_mov_b32 s29, s34
-; SI-NEXT: s_mov_b32 s16, s69
-; SI-NEXT: s_mov_b32 s69, s84
-; SI-NEXT: s_mov_b32 s84, s78
-; SI-NEXT: s_mov_b32 s78, s45
-; SI-NEXT: s_mov_b32 s28, s93
+; SI-NEXT: s_mov_b32 s16, s48
; SI-NEXT: s_mov_b32 s93, s44
-; SI-NEXT: s_lshr_b32 s47, s57, 16
+; SI-NEXT: s_mov_b32 s44, s26
+; SI-NEXT: s_mov_b32 s26, s85
+; SI-NEXT: s_mov_b32 s28, s98
+; SI-NEXT: s_mov_b32 s98, s45
+; SI-NEXT: s_lshr_b32 s47, s76, 16
; SI-NEXT: s_lshr_b32 s57, s77, 16
-; SI-NEXT: s_lshr_b32 s59, vcc_lo, 16
-; SI-NEXT: s_lshr_b32 s61, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s59, s23, 16
+; SI-NEXT: s_mov_b32 s23, s34
+; SI-NEXT: s_mov_b32 s34, s55
+; SI-NEXT: v_readlane_b32 s55, v44, 10
+; SI-NEXT: s_lshr_b32 s61, s21, 16
+; SI-NEXT: s_mov_b32 s21, s50
+; SI-NEXT: s_mov_b32 s27, s67
+; SI-NEXT: v_readlane_b32 s50, v44, 12
+; SI-NEXT: v_readlane_b32 s67, v44, 11
; SI-NEXT: s_lshr_b32 s63, s20, 16
-; SI-NEXT: s_mov_b32 s20, s67
-; SI-NEXT: s_mov_b32 s34, s68
-; SI-NEXT: v_readlane_b32 s68, v44, 11
+; SI-NEXT: s_mov_b32 s20, s95
+; SI-NEXT: s_mov_b32 s85, s35
+; SI-NEXT: s_mov_b32 s35, s52
; SI-NEXT: s_lshr_b32 s73, s18, 16
-; SI-NEXT: s_mov_b32 s18, s88
-; SI-NEXT: s_mov_b32 s27, s95
-; SI-NEXT: s_mov_b32 s95, s51
-; SI-NEXT: s_mov_b32 s51, s94
-; SI-NEXT: s_mov_b32 s88, s82
-; SI-NEXT: s_mov_b32 s82, s91
-; SI-NEXT: s_mov_b32 s91, s39
-; SI-NEXT: s_mov_b32 s39, s70
+; SI-NEXT: s_mov_b32 s18, s96
+; SI-NEXT: s_mov_b32 s96, s31
+; SI-NEXT: s_mov_b32 s31, s53
+; SI-NEXT: s_mov_b32 s53, s69
+; SI-NEXT: s_mov_b32 s69, s24
+; SI-NEXT: s_mov_b32 s29, s91
+; SI-NEXT: s_mov_b32 s91, s38
+; SI-NEXT: s_mov_b32 s38, s65
+; SI-NEXT: s_mov_b32 s65, s82
; SI-NEXT: s_lshr_b32 s75, s19, 16
; SI-NEXT: s_mov_b32 s19, s90
-; SI-NEXT: s_mov_b32 s90, s38
-; SI-NEXT: s_mov_b32 s38, s53
-; SI-NEXT: s_mov_b32 s53, s31
-; SI-NEXT: s_mov_b32 s44, s92
-; SI-NEXT: s_mov_b32 s92, s49
-; SI-NEXT: s_mov_b32 s49, s55
-; SI-NEXT: s_mov_b32 s55, s22
-; SI-NEXT: s_mov_b32 s31, s80
-; SI-NEXT: s_mov_b32 s22, s97
+; SI-NEXT: s_mov_b32 s90, s39
+; SI-NEXT: s_mov_b32 s39, s54
+; SI-NEXT: s_mov_b32 s54, s83
+; SI-NEXT: s_mov_b32 s83, s87
+; SI-NEXT: s_mov_b32 s87, s22
+; SI-NEXT: s_mov_b32 s22, s56
; SI-NEXT: s_lshr_b32 s45, s17, 16
; SI-NEXT: s_mov_b32 s17, s89
-; SI-NEXT: s_mov_b32 s89, s48
-; SI-NEXT: s_mov_b32 s48, s65
-; SI-NEXT: s_mov_b32 s65, s81
-; SI-NEXT: s_mov_b32 s81, s86
-; SI-NEXT: s_mov_b32 s86, s56
-; SI-NEXT: s_mov_b32 s56, s76
+; SI-NEXT: s_mov_b32 s89, s36
+; SI-NEXT: s_mov_b32 s36, s66
+; SI-NEXT: s_mov_b32 s66, s70
+; SI-NEXT: s_mov_b32 s70, s97
; SI-NEXT: s_mov_b64 s[76:77], 0
; SI-NEXT: s_branch .LBB107_3
; SI-NEXT: .LBB107_2:
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: s_mov_b64 s[76:77], -1
-; SI-NEXT: v_writelane_b32 v44, s4, 7
-; SI-NEXT: v_writelane_b32 v44, s5, 8
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr47
; SI-NEXT: ; implicit-def: $sgpr40
; SI-NEXT: ; implicit-def: $sgpr46
; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr92
; SI-NEXT: ; implicit-def: $sgpr59
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr58
@@ -82823,215 +83914,216 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr74
; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: v_writelane_b32 v44, s4, 9
-; SI-NEXT: v_writelane_b32 v44, s5, 10
-; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: .LBB107_3: ; %Flow
-; SI-NEXT: s_andn2_b64 vcc, exec, s[76:77]
-; SI-NEXT: v_readlane_b32 s76, v44, 7
-; SI-NEXT: v_readlane_b32 s77, v44, 8
-; SI-NEXT: s_cbranch_vccnz .LBB107_5
+; SI-NEXT: s_and_b64 s[76:77], s[76:77], exec
+; SI-NEXT: s_cselect_b32 s76, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s76, 1
+; SI-NEXT: s_cbranch_scc1 .LBB107_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_and_b32 s4, s21, 0xff
-; SI-NEXT: s_lshl_b32 s5, s25, 8
-; SI-NEXT: s_add_i32 s78, s78, 3
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_and_b32 s4, s22, 0xff
+; SI-NEXT: s_lshl_b32 s5, s26, 8
+; SI-NEXT: s_add_i32 s98, s98, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s78, 0xff
+; SI-NEXT: s_and_b32 s6, s98, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s24, 24
+; SI-NEXT: s_lshl_b32 s5, s25, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s99, s99, 3
-; SI-NEXT: v_readlane_b32 s6, v44, 6
+; SI-NEXT: s_add_i32 s97, s30, 3
+; SI-NEXT: v_readlane_b32 s6, v44, 9
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s99, 0xff
+; SI-NEXT: s_and_b32 s5, s97, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s7, s86, 0xff
+; SI-NEXT: v_readlane_b32 s6, v44, 7
+; SI-NEXT: s_and_b32 s7, s84, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s6, s56, 24
+; SI-NEXT: s_lshl_b32 s6, s6, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s97, s79, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
+; SI-NEXT: v_readlane_b32 s8, v44, 6
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s97, 0xff
-; SI-NEXT: s_lshl_b32 s7, s36, 8
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_and_b32 s6, s87, 0xff
+; SI-NEXT: s_lshl_b32 s7, s44, 8
+; SI-NEXT: s_add_i32 s82, s8, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: v_readlane_b32 s7, v44, 5
-; SI-NEXT: s_and_b32 s8, s84, 0xff
+; SI-NEXT: v_readlane_b32 s7, v44, 8
+; SI-NEXT: s_and_b32 s8, s82, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s7, s8
-; SI-NEXT: s_add_i32 s83, s83, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s83, 0xff
-; SI-NEXT: s_lshl_b32 s8, s87, 8
-; SI-NEXT: s_add_i32 s80, s23, 3
+; SI-NEXT: v_readlane_b32 s7, v44, 5
+; SI-NEXT: s_add_i32 s81, s7, 3
+; SI-NEXT: s_and_b32 s7, s81, 0xff
+; SI-NEXT: s_lshl_b32 s8, s78, 8
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s9, s80, 0xff
+; SI-NEXT: s_and_b32 s9, s70, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s8, s22, 24
+; SI-NEXT: s_lshl_b32 s8, s99, 24
; SI-NEXT: s_lshl_b32 s9, s9, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: s_add_i32 s67, s67, 3
; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s68, 0xff
-; SI-NEXT: s_lshl_b32 s9, s69, 8
-; SI-NEXT: s_add_i32 s66, s66, 3
+; SI-NEXT: s_and_b32 s8, s67, 0xff
+; SI-NEXT: s_lshl_b32 s9, s71, 8
+; SI-NEXT: s_add_i32 s65, s65, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s10, s66, 0xff
+; SI-NEXT: s_and_b32 s10, s65, 0xff
; SI-NEXT: s_addk_i32 s8, 0x300
; SI-NEXT: s_lshl_b32 s9, s55, 24
; SI-NEXT: s_lshl_b32 s10, s10, 16
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s64, s64, 3
+; SI-NEXT: s_add_i32 s55, s69, 3
; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s64, 0xff
-; SI-NEXT: s_lshl_b32 s10, s81, 8
-; SI-NEXT: s_add_i32 s69, s98, 3
+; SI-NEXT: s_and_b32 s9, s55, 0xff
+; SI-NEXT: s_lshl_b32 s10, s83, 8
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s11, s69, 0xff
+; SI-NEXT: s_and_b32 s11, vcc_hi, 0xff
; SI-NEXT: s_addk_i32 s9, 0x300
-; SI-NEXT: s_lshl_b32 s10, s71, 24
+; SI-NEXT: s_lshl_b32 s10, vcc_lo, 24
; SI-NEXT: s_lshl_b32 s11, s11, 16
; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_add_i32 s55, s31, 3
+; SI-NEXT: s_add_i32 s54, s54, 3
; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s55, 0xff
-; SI-NEXT: s_lshl_b32 s11, s88, 8
-; SI-NEXT: s_add_i32 s53, s53, 3
+; SI-NEXT: s_and_b32 s10, s54, 0xff
+; SI-NEXT: s_lshl_b32 s11, s66, 8
+; SI-NEXT: s_add_i32 s52, s88, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s12, s53, 0xff
+; SI-NEXT: s_and_b32 s12, s52, 0xff
; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s11, s37, 24
+; SI-NEXT: s_lshl_b32 s11, s50, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_add_i32 s51, s51, 3
+; SI-NEXT: s_add_i32 s50, s94, 3
; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s51, 0xff
-; SI-NEXT: s_lshl_b32 s12, s65, 8
+; SI-NEXT: s_and_b32 s11, s50, 0xff
+; SI-NEXT: s_lshl_b32 s12, s49, 8
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_lshl_b32 s12, s39, 24
-; SI-NEXT: s_add_i32 s39, s50, 3
-; SI-NEXT: s_and_b32 s13, s39, 0xff
+; SI-NEXT: s_lshl_b32 s12, s38, 24
+; SI-NEXT: s_add_i32 s38, s68, 3
+; SI-NEXT: s_and_b32 s13, s38, 0xff
; SI-NEXT: s_addk_i32 s11, 0x300
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_and_b32 s11, s11, 0xffff
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: s_add_i32 s50, s96, 3
+; SI-NEXT: s_add_i32 s49, s64, 3
; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s50, 0xff
-; SI-NEXT: s_lshl_b32 s13, s16, 8
-; SI-NEXT: s_add_i32 s37, s52, 3
+; SI-NEXT: s_and_b32 s12, s49, 0xff
+; SI-NEXT: s_lshl_b32 s13, s53, 8
+; SI-NEXT: s_add_i32 s36, s36, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s14, s37, 0xff
+; SI-NEXT: s_and_b32 s14, s36, 0xff
; SI-NEXT: s_addk_i32 s12, 0x300
; SI-NEXT: s_lshl_b32 s13, s34, 24
; SI-NEXT: s_lshl_b32 s14, s14, 16
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s34, s39, 3
; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s35, 0xff
-; SI-NEXT: s_lshl_b32 s14, s48, 8
-; SI-NEXT: v_readlane_b32 s15, v44, 4
+; SI-NEXT: s_and_b32 s13, s34, 0xff
+; SI-NEXT: s_lshl_b32 s14, s37, 8
+; SI-NEXT: s_add_i32 s95, s31, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_lshl_b32 s14, s95, 24
-; SI-NEXT: s_add_i32 s95, s15, 3
; SI-NEXT: s_and_b32 s15, s95, 0xff
; SI-NEXT: s_addk_i32 s13, 0x300
+; SI-NEXT: s_lshl_b32 s14, s86, 24
; SI-NEXT: s_lshl_b32 s15, s15, 16
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_or_b32 s14, s14, s15
-; SI-NEXT: s_add_i32 s34, s49, 3
+; SI-NEXT: s_add_i32 vcc_lo, s51, 3
; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s34, 0xff
-; SI-NEXT: s_lshl_b32 s15, s38, 8
+; SI-NEXT: s_and_b32 s14, vcc_lo, 0xff
+; SI-NEXT: s_lshl_b32 s15, s35, 8
+; SI-NEXT: s_add_i32 s93, s93, 3
; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_lshl_b32 s15, s93, 24
-; SI-NEXT: s_add_i32 s93, s30, 3
-; SI-NEXT: s_and_b32 s21, s93, 0xff
+; SI-NEXT: s_and_b32 s22, s93, 0xff
; SI-NEXT: s_addk_i32 s14, 0x300
-; SI-NEXT: s_lshl_b32 s21, s21, 16
+; SI-NEXT: s_lshl_b32 s15, s28, 24
+; SI-NEXT: s_lshl_b32 s22, s22, 16
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_or_b32 s15, s15, s21
+; SI-NEXT: s_or_b32 s15, s15, s22
; SI-NEXT: s_add_i32 s91, s91, 3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s15, s91, 0xff
-; SI-NEXT: s_lshl_b32 s21, s92, 8
-; SI-NEXT: s_add_i32 s89, s89, 3
-; SI-NEXT: s_or_b32 s15, s21, s15
-; SI-NEXT: s_and_b32 s22, s89, 0xff
+; SI-NEXT: s_lshl_b32 s22, s16, 8
+; SI-NEXT: s_or_b32 s15, s22, s15
+; SI-NEXT: s_lshl_b32 s22, s89, 24
+; SI-NEXT: s_add_i32 s89, s90, 3
+; SI-NEXT: s_and_b32 s40, s89, 0xff
; SI-NEXT: s_addk_i32 s15, 0x300
-; SI-NEXT: s_lshl_b32 s21, s20, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
-; SI-NEXT: s_add_i32 s16, s44, 3
+; SI-NEXT: s_lshl_b32 s40, s40, 16
+; SI-NEXT: v_readlane_b32 s16, v44, 0
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_or_b32 s21, s21, s22
-; SI-NEXT: s_add_i32 s24, s29, 3
+; SI-NEXT: s_or_b32 s22, s22, s40
+; SI-NEXT: s_add_i32 s24, s85, 3
+; SI-NEXT: v_readlane_b32 s25, v44, 1
+; SI-NEXT: s_add_i32 s28, s16, 3
+; SI-NEXT: v_readlane_b32 s16, v44, 4
+; SI-NEXT: s_or_b32 s15, s22, s15
+; SI-NEXT: s_and_b32 s22, s24, 0xff
+; SI-NEXT: s_lshl_b32 s24, s27, 8
+; SI-NEXT: s_add_i32 s26, s25, 3
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_and_b32 s25, s26, 0xff
; SI-NEXT: s_and_b32 s16, s16, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s15, s21, s15
-; SI-NEXT: s_and_b32 s21, s24, 0xff
-; SI-NEXT: s_lshl_b32 s22, s85, 8
-; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_addk_i32 s22, 0x300
+; SI-NEXT: s_lshl_b32 s24, s96, 24
+; SI-NEXT: s_lshl_b32 s25, s25, 16
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_and_b32 s20, s26, 0xff
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s19, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_addk_i32 s21, 0x300
-; SI-NEXT: s_lshl_b32 s22, s27, 24
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_or_b32 s22, s24, s22
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_or_b32 s22, s22, s20
+; SI-NEXT: s_add_i32 s40, s22, 0x3000000
+; SI-NEXT: s_and_b32 s22, s28, 0xff
+; SI-NEXT: s_lshl_b32 s24, s29, 8
+; SI-NEXT: s_add_i32 s79, s79, 3
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: v_readlane_b32 s18, v44, 3
+; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_and_b32 s24, s79, 0xff
; SI-NEXT: s_add_i32 s42, s16, 0x3000000
-; SI-NEXT: v_readlane_b32 s16, v44, 3
-; SI-NEXT: s_add_i32 s40, s21, 0x3000000
-; SI-NEXT: s_and_b32 s21, s28, 0xff
-; SI-NEXT: s_lshl_b32 s22, s82, 8
-; SI-NEXT: s_add_i32 s79, s90, 3
-; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: v_readlane_b32 s17, v44, 2
-; SI-NEXT: v_readlane_b32 s18, v44, 1
-; SI-NEXT: s_or_b32 s21, s22, s21
-; SI-NEXT: s_and_b32 s22, s79, 0xff
-; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: s_lshl_b32 s17, s17, 8
+; SI-NEXT: s_and_b32 s16, s20, 0xff
+; SI-NEXT: s_lshl_b32 s17, s21, 8
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_addk_i32 s21, 0x300
-; SI-NEXT: s_lshl_b32 s20, s54, 24
-; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_addk_i32 s22, 0x300
+; SI-NEXT: s_lshl_b32 s23, s23, 24
+; SI-NEXT: s_lshl_b32 s24, s24, 16
; SI-NEXT: s_or_b32 s16, s17, s16
-; SI-NEXT: v_readlane_b32 s17, v44, 0
+; SI-NEXT: v_readlane_b32 s17, v44, 2
; SI-NEXT: s_and_b32 s18, s18, 0xff
-; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_or_b32 s20, s20, s22
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_lshl_b32 s18, s18, 16
-; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_or_b32 s22, s23, s22
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
@@ -83044,20 +84136,19 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s11, s11, 0x3000000
; SI-NEXT: s_add_i32 s12, s12, 0x3000000
; SI-NEXT: s_add_i32 s13, s13, 0x3000000
+; SI-NEXT: s_add_i32 s41, s22, 0x3000000
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s14, s14, 0x3000000
; SI-NEXT: s_add_i32 s15, s15, 0x3000000
-; SI-NEXT: s_add_i32 s41, s20, 0x3000000
-; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s43, s16, 0x3000000
; SI-NEXT: s_lshr_b64 s[46:47], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v44, s16, 9
+; SI-NEXT: s_lshr_b64 s[80:81], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
; SI-NEXT: s_lshr_b32 s47, s43, 16
; SI-NEXT: s_lshr_b32 s57, s41, 16
; SI-NEXT: s_lshr_b32 s59, s15, 16
@@ -83066,13 +84157,12 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_lshr_b32 s73, s9, 16
; SI-NEXT: s_lshr_b32 s75, s7, 16
; SI-NEXT: s_lshr_b32 s45, s5, 16
-; SI-NEXT: v_writelane_b32 v44, s17, 10
; SI-NEXT: .LBB107_5: ; %end
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s16, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s76, 16
+; SI-NEXT: s_lshl_b32 s17, s80, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s43, 0xffff
; SI-NEXT: s_lshl_b32 s18, s47, 16
@@ -83083,9 +84173,8 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s19, s41, 0xffff
; SI-NEXT: s_lshl_b32 s20, s57, 16
; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: v_readlane_b32 s20, v44, 9
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_lshl_b32 s20, s92, 16
; SI-NEXT: s_or_b32 s14, s14, s20
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_lshl_b32 s20, s59, 16
@@ -83121,7 +84210,6 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s20, s45, 16
; SI-NEXT: s_or_b32 s5, s5, s20
; SI-NEXT: v_readlane_b32 s30, v43, 34
-; SI-NEXT: v_readlane_b32 s21, v44, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -83237,7 +84325,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB107_4
+; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -83329,8 +84417,17 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v33, v28, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB107_3
-; VI-NEXT: .LBB107_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB107_3
+; VI-NEXT: .LBB107_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB107_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB107_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_u32_e32 v0, vcc, 3, v34
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v8, v0, v31, s4
@@ -83490,12 +84587,9 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB107_3: ; %end
+; VI-NEXT: .LBB107_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB107_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB107_2
;
; GFX9-LABEL: bitcast_v64i8_to_v32f16_scalar:
; GFX9: ; %bb.0:
@@ -83554,7 +84648,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
@@ -83660,8 +84754,17 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB107_3
-; GFX9-NEXT: .LBB107_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB107_3
+; GFX9-NEXT: .LBB107_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB107_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -83818,12 +84921,9 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
-; GFX9-NEXT: .LBB107_3: ; %end
+; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB107_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB107_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32f16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -83880,7 +84980,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
@@ -83959,9 +85059,17 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v18.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-TRUE16-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB107_3
+; GFX11-TRUE16-NEXT: .LBB107_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -84078,12 +85186,9 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: .LBB107_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB107_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB107_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v32f16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -84140,7 +85245,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s58, 0xff
@@ -84233,9 +85338,17 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v16, 16, v17
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, s77
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-FAKE16-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB107_3
+; GFX11-FAKE16-NEXT: .LBB107_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -84368,12 +85481,9 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v16, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v14, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v22
-; GFX11-FAKE16-NEXT: .LBB107_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB107_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB107_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -87490,7 +88600,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s43
; SI-NEXT: v_mul_f32_e64 v34, 1.0, s42
; SI-NEXT: ; implicit-def: $vgpr41 : SGPR spill to VGPR lane
-; SI-NEXT: s_cbranch_scc0 .LBB109_4
+; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_lshr_b32 s5, s4, 16
@@ -87498,19 +88608,19 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s72, v2
-; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
-; SI-NEXT: s_mov_b32 s75, s76
-; SI-NEXT: s_lshr_b64 s[4:5], s[74:75], 24
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v41, s4, 0
-; SI-NEXT: v_writelane_b32 v41, s5, 1
; SI-NEXT: v_readfirstlane_b32 s4, v7
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_lshr_b64 s[60:61], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v5
; SI-NEXT: s_lshr_b32 s59, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s58, v6
+; SI-NEXT: s_lshr_b64 s[62:63], s[58:59], 16
+; SI-NEXT: s_mov_b32 s61, s62
+; SI-NEXT: s_lshr_b64 s[4:5], s[60:61], 24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v41, s4, 0
+; SI-NEXT: v_writelane_b32 v41, s5, 1
; SI-NEXT: v_readfirstlane_b32 s4, v11
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -87546,7 +88656,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s4, v37
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v33
-; SI-NEXT: v_readfirstlane_b32 s58, v6
+; SI-NEXT: v_readfirstlane_b32 s72, v2
; SI-NEXT: v_readfirstlane_b32 s44, v10
; SI-NEXT: v_readfirstlane_b32 s24, v14
; SI-NEXT: v_readfirstlane_b32 s40, v18
@@ -87554,23 +88664,23 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s10, v28
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v34
-; SI-NEXT: s_lshr_b64 s[62:63], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[44:45], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[22:23], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[28:29], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
-; SI-NEXT: s_mov_b32 s61, s62
+; SI-NEXT: s_mov_b32 s75, s76
; SI-NEXT: s_mov_b32 s47, s56
; SI-NEXT: s_mov_b32 s27, s42
; SI-NEXT: s_mov_b32 s17, s22
; SI-NEXT: s_mov_b32 s21, s28
; SI-NEXT: s_mov_b32 s13, s14
; SI-NEXT: s_mov_b32 s7, s8
-; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 24
+; SI-NEXT: s_lshr_b64 s[90:91], s[74:75], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[74:75], 8
-; SI-NEXT: s_lshr_b64 s[90:91], s[60:61], 24
; SI-NEXT: s_lshr_b64 s[94:95], s[60:61], 16
; SI-NEXT: s_lshr_b64 s[30:31], s[60:61], 8
; SI-NEXT: s_lshr_b64 s[34:35], s[46:47], 24
@@ -87607,8 +88717,83 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshr_b64 s[82:83], s[6:7], 24
; SI-NEXT: s_lshr_b64 s[84:85], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[86:87], s[6:7], 8
-; SI-NEXT: s_cbranch_execnz .LBB109_3
-; SI-NEXT: .LBB109_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 vcc, 0
+; SI-NEXT: s_branch .LBB109_3
+; SI-NEXT: .LBB109_2:
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: s_mov_b64 vcc, -1
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v41, s4, 0
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $vgpr48
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: v_writelane_b32 v41, s5, 1
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: ; implicit-def: $sgpr59
+; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr18
+; SI-NEXT: ; implicit-def: $sgpr45
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $sgpr26
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr17
+; SI-NEXT: ; implicit-def: $sgpr25
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $sgpr16
+; SI-NEXT: ; implicit-def: $sgpr66
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $vgpr32
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $vgpr26
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $sgpr41
+; SI-NEXT: ; implicit-def: $sgpr20
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr70
+; SI-NEXT: ; implicit-def: $sgpr68
+; SI-NEXT: ; implicit-def: $sgpr28
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr96
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr84
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr5
+; SI-NEXT: .LBB109_3: ; %Flow
+; SI-NEXT: s_and_b64 vcc, vcc, exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB109_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v36
; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v37
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
@@ -87689,68 +88874,68 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshr_b32 s45, s4, 16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: s_lshr_b32 s59, s4, 16
; SI-NEXT: v_readfirstlane_b32 s58, v8
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_readfirstlane_b32 s4, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; SI-NEXT: s_lshr_b64 s[60:61], s[58:59], 16
; SI-NEXT: s_lshr_b32 s59, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s58, v6
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s72, v4
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
-; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s72, v2
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[58:59], 16
+; SI-NEXT: s_lshr_b32 s73, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s72, v4
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: v_readfirstlane_b32 s10, v28
; SI-NEXT: v_readfirstlane_b32 s18, v22
; SI-NEXT: v_readfirstlane_b32 s16, v20
; SI-NEXT: v_readfirstlane_b32 s40, v18
; SI-NEXT: v_readfirstlane_b32 s24, v14
; SI-NEXT: v_readfirstlane_b32 s44, v10
-; SI-NEXT: v_readfirstlane_b32 s58, v6
-; SI-NEXT: s_mov_b32 s75, s76
+; SI-NEXT: s_mov_b32 s61, s62
+; SI-NEXT: s_lshr_b64 s[74:75], s[72:73], 16
+; SI-NEXT: s_lshr_b32 s73, s4, 16
+; SI-NEXT: v_readfirstlane_b32 s72, v2
; SI-NEXT: s_lshr_b64 s[14:15], s[10:11], 16
; SI-NEXT: s_lshr_b64 s[28:29], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[16:17], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[22:23], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[78:79], s[74:75], 24
+; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[60:61], 24
; SI-NEXT: s_mov_b32 s7, s8
; SI-NEXT: s_mov_b32 s13, s14
; SI-NEXT: s_mov_b32 s21, s28
; SI-NEXT: s_mov_b32 s17, s22
; SI-NEXT: s_mov_b32 s27, s42
; SI-NEXT: s_mov_b32 s47, s56
-; SI-NEXT: s_mov_b32 s61, s62
+; SI-NEXT: s_mov_b32 s75, s76
; SI-NEXT: v_writelane_b32 v41, s78, 0
-; SI-NEXT: v_writelane_b32 v41, s79, 1
-; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[74:75], 24
+; SI-NEXT: s_lshr_b64 s[90:91], s[74:75], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[74:75], 8
-; SI-NEXT: s_lshr_b64 s[90:91], s[60:61], 24
+; SI-NEXT: v_writelane_b32 v41, s79, 1
; SI-NEXT: s_lshr_b64 s[94:95], s[60:61], 16
; SI-NEXT: s_lshr_b64 s[30:31], s[60:61], 8
; SI-NEXT: s_lshr_b64 s[34:35], s[46:47], 24
@@ -87787,13 +88972,12 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshr_b64 s[82:83], s[6:7], 24
; SI-NEXT: s_lshr_b64 s[84:85], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[86:87], s[6:7], 8
-; SI-NEXT: .LBB109_3: ; %end
+; SI-NEXT: .LBB109_5: ; %end
; SI-NEXT: s_and_b32 s7, s74, 0xff
; SI-NEXT: s_lshl_b32 s13, s92, 8
; SI-NEXT: s_or_b32 s7, s7, s13
-; SI-NEXT: s_and_b32 s13, s88, 0xff
-; SI-NEXT: v_readlane_b32 s74, v41, 0
-; SI-NEXT: s_lshl_b32 s24, s74, 24
+; SI-NEXT: s_and_b32 s13, s90, 0xff
+; SI-NEXT: s_lshl_b32 s24, s88, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
; SI-NEXT: s_or_b32 s13, s24, s13
; SI-NEXT: s_and_b32 s7, s7, 0xffff
@@ -87814,8 +88998,9 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_lshl_b32 s13, s30, 8
; SI-NEXT: s_or_b32 s7, s7, s13
; SI-NEXT: s_and_b32 s13, s94, 0xff
+; SI-NEXT: v_readlane_b32 s60, v41, 0
; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_lshl_b32 s23, s90, 24
+; SI-NEXT: s_lshl_b32 s23, s60, 24
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_or_b32 s13, s23, s13
; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v0
@@ -87986,7 +89171,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_or_b32_e32 v1, s4, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, 60, v0
; SI-NEXT: v_readlane_b32 s30, v40, 34
-; SI-NEXT: v_readlane_b32 s75, v41, 1
+; SI-NEXT: v_readlane_b32 s61, v41, 1
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: v_readlane_b32 s31, v40, 35
; SI-NEXT: v_readlane_b32 s99, v40, 33
@@ -88029,75 +89214,6 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB109_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v41, s4, 0
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: v_writelane_b32 v41, s5, 1
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $vgpr48
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr59
-; SI-NEXT: ; implicit-def: $vgpr39
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr45
-; SI-NEXT: ; implicit-def: $vgpr38
-; SI-NEXT: ; implicit-def: $sgpr26
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr17
-; SI-NEXT: ; implicit-def: $sgpr25
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: ; implicit-def: $sgpr16
-; SI-NEXT: ; implicit-def: $sgpr66
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $vgpr30
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr25
-; SI-NEXT: ; implicit-def: $sgpr41
-; SI-NEXT: ; implicit-def: $sgpr20
-; SI-NEXT: ; implicit-def: $sgpr80
-; SI-NEXT: ; implicit-def: $sgpr70
-; SI-NEXT: ; implicit-def: $sgpr68
-; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr98
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr86
-; SI-NEXT: ; implicit-def: $sgpr84
-; SI-NEXT: ; implicit-def: $sgpr82
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr5
-; SI-NEXT: s_branch .LBB109_2
;
; VI-LABEL: bitcast_v32bf16_to_v64i8_scalar:
; VI: ; %bb.0:
@@ -88136,36 +89252,34 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_writelane_b32 v63, s55, 13
; VI-NEXT: v_writelane_b32 v63, s64, 14
; VI-NEXT: v_writelane_b32 v63, s65, 15
-; VI-NEXT: v_writelane_b32 v63, s66, 16
-; VI-NEXT: v_writelane_b32 v63, s67, 17
-; VI-NEXT: v_writelane_b32 v63, s30, 18
-; VI-NEXT: v_writelane_b32 v63, s31, 19
+; VI-NEXT: v_writelane_b32 v63, s30, 16
+; VI-NEXT: v_writelane_b32 v63, s31, 17
; VI-NEXT: v_readfirstlane_b32 s4, v3
; VI-NEXT: v_readfirstlane_b32 s5, v2
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v1
-; VI-NEXT: s_cbranch_scc0 .LBB109_3
+; VI-NEXT: s_cbranch_scc0 .LBB109_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_lshr_b32 s65, s5, 24
-; VI-NEXT: s_lshr_b32 s64, s5, 16
-; VI-NEXT: s_lshr_b32 s55, s5, 8
-; VI-NEXT: s_lshr_b32 s67, s4, 16
-; VI-NEXT: s_lshr_b32 s66, s4, 8
-; VI-NEXT: s_lshr_b32 s52, s29, 24
-; VI-NEXT: s_lshr_b32 s51, s29, 16
-; VI-NEXT: s_lshr_b32 s50, s29, 8
-; VI-NEXT: s_lshr_b32 s54, s28, 16
-; VI-NEXT: s_lshr_b32 s53, s28, 8
-; VI-NEXT: s_lshr_b32 s39, s27, 24
-; VI-NEXT: s_lshr_b32 s38, s27, 16
-; VI-NEXT: s_lshr_b32 s37, s27, 8
-; VI-NEXT: s_lshr_b32 s49, s26, 16
-; VI-NEXT: s_lshr_b32 s48, s26, 8
-; VI-NEXT: s_lshr_b32 s34, s25, 24
-; VI-NEXT: s_lshr_b32 s31, s25, 16
-; VI-NEXT: s_lshr_b32 s30, s25, 8
-; VI-NEXT: s_lshr_b32 s36, s24, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 8
+; VI-NEXT: s_lshr_b32 s55, s5, 24
+; VI-NEXT: s_lshr_b32 s54, s5, 16
+; VI-NEXT: s_lshr_b32 s53, s5, 8
+; VI-NEXT: s_lshr_b32 s65, s4, 16
+; VI-NEXT: s_lshr_b32 s64, s4, 8
+; VI-NEXT: s_lshr_b32 s50, s29, 24
+; VI-NEXT: s_lshr_b32 s49, s29, 16
+; VI-NEXT: s_lshr_b32 s48, s29, 8
+; VI-NEXT: s_lshr_b32 s52, s28, 16
+; VI-NEXT: s_lshr_b32 s51, s28, 8
+; VI-NEXT: s_lshr_b32 s37, s27, 24
+; VI-NEXT: s_lshr_b32 s36, s27, 16
+; VI-NEXT: s_lshr_b32 s35, s27, 8
+; VI-NEXT: s_lshr_b32 s39, s26, 16
+; VI-NEXT: s_lshr_b32 s38, s26, 8
+; VI-NEXT: s_lshr_b32 s30, s25, 24
+; VI-NEXT: s_lshr_b32 vcc_hi, s25, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s25, 8
+; VI-NEXT: s_lshr_b32 s34, s24, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 8
; VI-NEXT: s_lshr_b32 s89, s23, 24
; VI-NEXT: s_lshr_b32 s88, s23, 16
; VI-NEXT: s_lshr_b32 s79, s23, 8
@@ -88194,8 +89308,64 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; VI-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB109_4
-; VI-NEXT: .LBB109_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[46:47], 0
+; VI-NEXT: s_branch .LBB109_3
+; VI-NEXT: .LBB109_2:
+; VI-NEXT: s_mov_b64 s[46:47], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr38
+; VI-NEXT: ; implicit-def: $sgpr39
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr51
+; VI-NEXT: ; implicit-def: $sgpr52
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr48
+; VI-NEXT: ; implicit-def: $sgpr49
+; VI-NEXT: ; implicit-def: $sgpr50
+; VI-NEXT: ; implicit-def: $sgpr64
+; VI-NEXT: ; implicit-def: $sgpr65
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr53
+; VI-NEXT: ; implicit-def: $sgpr54
+; VI-NEXT: ; implicit-def: $sgpr55
+; VI-NEXT: .LBB109_3: ; %Flow
+; VI-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; VI-NEXT: s_cselect_b32 s7, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s7, 1
+; VI-NEXT: s_cbranch_scc1 .LBB109_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s6, s17, 16
; VI-NEXT: v_mov_b32_e32 v24, 0x40c00000
; VI-NEXT: v_add_f32_e32 v1, s6, v24
@@ -88543,80 +89713,30 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v62, 8, v1
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v6, 8, v2
-; VI-NEXT: s_branch .LBB109_5
-; VI-NEXT: .LBB109_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr48
-; VI-NEXT: ; implicit-def: $sgpr49
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr53
-; VI-NEXT: ; implicit-def: $sgpr54
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr50
-; VI-NEXT: ; implicit-def: $sgpr51
-; VI-NEXT: ; implicit-def: $sgpr52
-; VI-NEXT: ; implicit-def: $sgpr66
-; VI-NEXT: ; implicit-def: $sgpr67
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr55
-; VI-NEXT: ; implicit-def: $sgpr64
-; VI-NEXT: ; implicit-def: $sgpr65
-; VI-NEXT: s_branch .LBB109_2
-; VI-NEXT: .LBB109_4:
-; VI-NEXT: v_mov_b32_e32 v1, s65
+; VI-NEXT: s_branch .LBB109_6
+; VI-NEXT: .LBB109_5:
+; VI-NEXT: v_mov_b32_e32 v1, s55
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v1, s64
-; VI-NEXT: v_mov_b32_e32 v12, s67
-; VI-NEXT: v_mov_b32_e32 v15, s66
+; VI-NEXT: v_mov_b32_e32 v1, s54
+; VI-NEXT: v_mov_b32_e32 v12, s65
+; VI-NEXT: v_mov_b32_e32 v15, s64
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; VI-NEXT: v_mov_b32_e32 v9, s55
-; VI-NEXT: v_mov_b32_e32 v33, s54
-; VI-NEXT: v_mov_b32_e32 v34, s53
-; VI-NEXT: v_mov_b32_e32 v18, s52
-; VI-NEXT: v_mov_b32_e32 v21, s51
-; VI-NEXT: v_mov_b32_e32 v32, s50
-; VI-NEXT: v_mov_b32_e32 v38, s49
-; VI-NEXT: v_mov_b32_e32 v39, s48
-; VI-NEXT: v_mov_b32_e32 v35, s39
-; VI-NEXT: v_mov_b32_e32 v36, s38
-; VI-NEXT: v_mov_b32_e32 v37, s37
-; VI-NEXT: v_mov_b32_e32 v51, s36
-; VI-NEXT: v_mov_b32_e32 v52, s35
-; VI-NEXT: v_mov_b32_e32 v48, s34
-; VI-NEXT: v_mov_b32_e32 v49, s31
-; VI-NEXT: v_mov_b32_e32 v50, s30
+; VI-NEXT: v_mov_b32_e32 v9, s53
+; VI-NEXT: v_mov_b32_e32 v33, s52
+; VI-NEXT: v_mov_b32_e32 v34, s51
+; VI-NEXT: v_mov_b32_e32 v18, s50
+; VI-NEXT: v_mov_b32_e32 v21, s49
+; VI-NEXT: v_mov_b32_e32 v32, s48
+; VI-NEXT: v_mov_b32_e32 v38, s39
+; VI-NEXT: v_mov_b32_e32 v39, s38
+; VI-NEXT: v_mov_b32_e32 v35, s37
+; VI-NEXT: v_mov_b32_e32 v36, s36
+; VI-NEXT: v_mov_b32_e32 v37, s35
+; VI-NEXT: v_mov_b32_e32 v51, s34
+; VI-NEXT: v_mov_b32_e32 v52, s31
+; VI-NEXT: v_mov_b32_e32 v48, s30
+; VI-NEXT: v_mov_b32_e32 v49, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v50, vcc_lo
; VI-NEXT: v_mov_b32_e32 v40, s91
; VI-NEXT: v_mov_b32_e32 v41, s90
; VI-NEXT: v_mov_b32_e32 v53, s89
@@ -88661,7 +89781,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s10
; VI-NEXT: v_mov_b32_e32 v30, s8
; VI-NEXT: v_mov_b32_e32 v31, s6
-; VI-NEXT: .LBB109_5: ; %end
+; VI-NEXT: .LBB109_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v3, v3, v31, s4
; VI-NEXT: v_perm_b32 v2, v2, v6, s4
@@ -88756,10 +89876,8 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v1, v22, v9, s4
; VI-NEXT: v_add_u32_e32 v0, vcc, 60, v0
-; VI-NEXT: v_readlane_b32 s30, v63, 18
-; VI-NEXT: v_readlane_b32 s31, v63, 19
-; VI-NEXT: v_readlane_b32 s67, v63, 17
-; VI-NEXT: v_readlane_b32 s66, v63, 16
+; VI-NEXT: v_readlane_b32 s30, v63, 16
+; VI-NEXT: v_readlane_b32 s31, v63, 17
; VI-NEXT: v_readlane_b32 s65, v63, 15
; VI-NEXT: v_readlane_b32 s64, v63, 14
; VI-NEXT: v_readlane_b32 s55, v63, 13
@@ -88835,55 +89953,53 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_writelane_b32 v63, s51, 9
; GFX9-NEXT: v_writelane_b32 v63, s52, 10
; GFX9-NEXT: v_writelane_b32 v63, s53, 11
-; GFX9-NEXT: v_writelane_b32 v63, s54, 12
-; GFX9-NEXT: v_writelane_b32 v63, s55, 13
-; GFX9-NEXT: v_writelane_b32 v63, s30, 14
-; GFX9-NEXT: v_writelane_b32 v63, s31, 15
+; GFX9-NEXT: v_writelane_b32 v63, s30, 12
+; GFX9-NEXT: v_writelane_b32 v63, s31, 13
; GFX9-NEXT: v_readfirstlane_b32 s4, v3
; GFX9-NEXT: v_readfirstlane_b32 s5, v2
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB109_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s91, s5, 24
-; GFX9-NEXT: s_lshr_b32 s54, s5, 16
+; GFX9-NEXT: s_lshr_b32 s52, s5, 16
; GFX9-NEXT: s_lshr_b32 s93, s5, 8
-; GFX9-NEXT: s_lshr_b32 s55, s4, 16
+; GFX9-NEXT: s_lshr_b32 s53, s4, 16
; GFX9-NEXT: s_lshr_b32 s95, s4, 8
; GFX9-NEXT: s_lshr_b32 s88, s29, 24
-; GFX9-NEXT: s_lshr_b32 s52, s29, 16
+; GFX9-NEXT: s_lshr_b32 s50, s29, 16
; GFX9-NEXT: s_lshr_b32 s90, s29, 8
-; GFX9-NEXT: s_lshr_b32 s53, s28, 16
+; GFX9-NEXT: s_lshr_b32 s51, s28, 16
; GFX9-NEXT: s_lshr_b32 s94, s28, 8
; GFX9-NEXT: s_lshr_b32 s77, s27, 24
-; GFX9-NEXT: s_lshr_b32 s50, s27, 16
+; GFX9-NEXT: s_lshr_b32 s48, s27, 16
; GFX9-NEXT: s_lshr_b32 s79, s27, 8
-; GFX9-NEXT: s_lshr_b32 s51, s26, 16
+; GFX9-NEXT: s_lshr_b32 s49, s26, 16
; GFX9-NEXT: s_lshr_b32 s92, s26, 8
; GFX9-NEXT: s_lshr_b32 s74, s25, 24
-; GFX9-NEXT: s_lshr_b32 s48, s25, 16
+; GFX9-NEXT: s_lshr_b32 s38, s25, 16
; GFX9-NEXT: s_lshr_b32 s76, s25, 8
-; GFX9-NEXT: s_lshr_b32 s49, s24, 16
+; GFX9-NEXT: s_lshr_b32 s39, s24, 16
; GFX9-NEXT: s_lshr_b32 s89, s24, 8
; GFX9-NEXT: s_lshr_b32 s63, s23, 24
-; GFX9-NEXT: s_lshr_b32 s38, s23, 16
+; GFX9-NEXT: s_lshr_b32 s36, s23, 16
; GFX9-NEXT: s_lshr_b32 s73, s23, 8
-; GFX9-NEXT: s_lshr_b32 s39, s22, 16
+; GFX9-NEXT: s_lshr_b32 s37, s22, 16
; GFX9-NEXT: s_lshr_b32 s78, s22, 8
; GFX9-NEXT: s_lshr_b32 s60, s21, 24
-; GFX9-NEXT: s_lshr_b32 s36, s21, 16
+; GFX9-NEXT: s_lshr_b32 s34, s21, 16
; GFX9-NEXT: s_lshr_b32 s62, s21, 8
-; GFX9-NEXT: s_lshr_b32 s37, s20, 16
+; GFX9-NEXT: s_lshr_b32 s35, s20, 16
; GFX9-NEXT: s_lshr_b32 s75, s20, 8
; GFX9-NEXT: s_lshr_b32 s58, s19, 24
-; GFX9-NEXT: s_lshr_b32 s34, s19, 16
+; GFX9-NEXT: s_lshr_b32 s30, s19, 16
; GFX9-NEXT: s_lshr_b32 s59, s19, 8
-; GFX9-NEXT: s_lshr_b32 s35, s18, 16
+; GFX9-NEXT: s_lshr_b32 s31, s18, 16
; GFX9-NEXT: s_lshr_b32 s72, s18, 8
; GFX9-NEXT: s_lshr_b32 s56, s17, 24
-; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s17, 16
; GFX9-NEXT: s_lshr_b32 s57, s17, 8
-; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s16, 16
; GFX9-NEXT: s_lshr_b32 s61, s16, 8
; GFX9-NEXT: s_lshr_b64 s[44:45], s[4:5], 24
; GFX9-NEXT: s_lshr_b64 s[42:43], s[28:29], 24
@@ -88893,8 +90009,64 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: s_lshr_b64 s[10:11], s[20:21], 24
; GFX9-NEXT: s_lshr_b64 s[8:9], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[6:7], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB109_4
-; GFX9-NEXT: .LBB109_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[46:47], 0
+; GFX9-NEXT: s_branch .LBB109_3
+; GFX9-NEXT: .LBB109_2:
+; GFX9-NEXT: s_mov_b64 s[46:47], -1
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr35
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr34
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr37
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr36
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr39
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr38
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr49
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr48
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr51
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr50
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr53
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr52
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: .LBB109_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[46:47], s[46:47], exec
+; GFX9-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s7, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_and_b32 s6, s17, 0xffff0000
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v2, s6, v1
@@ -89250,91 +90422,41 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 24, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_branch .LBB109_5
-; GFX9-NEXT: .LBB109_3:
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr37
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr36
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr39
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr38
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr49
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr48
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr51
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr50
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr53
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr52
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr55
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr54
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: s_branch .LBB109_2
-; GFX9-NEXT: .LBB109_4:
+; GFX9-NEXT: s_branch .LBB109_6
+; GFX9-NEXT: .LBB109_5:
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v1, s30
+; GFX9-NEXT: v_mov_b32_e32 v1, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v56, s4
-; GFX9-NEXT: v_mov_b32_e32 v47, s55
+; GFX9-NEXT: v_mov_b32_e32 v47, s53
; GFX9-NEXT: v_mov_b32_e32 v46, s5
-; GFX9-NEXT: v_mov_b32_e32 v44, s54
+; GFX9-NEXT: v_mov_b32_e32 v44, s52
; GFX9-NEXT: v_mov_b32_e32 v45, s28
-; GFX9-NEXT: v_mov_b32_e32 v43, s53
+; GFX9-NEXT: v_mov_b32_e32 v43, s51
; GFX9-NEXT: v_mov_b32_e32 v42, s29
-; GFX9-NEXT: v_mov_b32_e32 v40, s52
+; GFX9-NEXT: v_mov_b32_e32 v40, s50
; GFX9-NEXT: v_mov_b32_e32 v41, s26
-; GFX9-NEXT: v_mov_b32_e32 v55, s51
+; GFX9-NEXT: v_mov_b32_e32 v55, s49
; GFX9-NEXT: v_mov_b32_e32 v54, s27
-; GFX9-NEXT: v_mov_b32_e32 v52, s50
+; GFX9-NEXT: v_mov_b32_e32 v52, s48
; GFX9-NEXT: v_mov_b32_e32 v53, s24
-; GFX9-NEXT: v_mov_b32_e32 v51, s49
+; GFX9-NEXT: v_mov_b32_e32 v51, s39
; GFX9-NEXT: v_mov_b32_e32 v49, s25
-; GFX9-NEXT: v_mov_b32_e32 v39, s48
+; GFX9-NEXT: v_mov_b32_e32 v39, s38
; GFX9-NEXT: v_mov_b32_e32 v48, s22
-; GFX9-NEXT: v_mov_b32_e32 v38, s39
+; GFX9-NEXT: v_mov_b32_e32 v38, s37
; GFX9-NEXT: v_mov_b32_e32 v34, s23
-; GFX9-NEXT: v_mov_b32_e32 v33, s38
+; GFX9-NEXT: v_mov_b32_e32 v33, s36
; GFX9-NEXT: v_mov_b32_e32 v50, s20
-; GFX9-NEXT: v_mov_b32_e32 v36, s37
+; GFX9-NEXT: v_mov_b32_e32 v36, s35
; GFX9-NEXT: v_mov_b32_e32 v28, s21
-; GFX9-NEXT: v_mov_b32_e32 v27, s36
+; GFX9-NEXT: v_mov_b32_e32 v27, s34
; GFX9-NEXT: v_mov_b32_e32 v35, s18
-; GFX9-NEXT: v_mov_b32_e32 v37, s35
+; GFX9-NEXT: v_mov_b32_e32 v37, s31
; GFX9-NEXT: v_mov_b32_e32 v32, s19
-; GFX9-NEXT: v_mov_b32_e32 v29, s34
+; GFX9-NEXT: v_mov_b32_e32 v29, s30
; GFX9-NEXT: v_mov_b32_e32 v30, s16
-; GFX9-NEXT: v_mov_b32_e32 v31, s31
+; GFX9-NEXT: v_mov_b32_e32 v31, vcc_hi
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v13, s95
; GFX9-NEXT: v_mov_b32_e32 v2, s91
@@ -89368,7 +90490,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v12, s10
; GFX9-NEXT: v_mov_b32_e32 v16, s8
; GFX9-NEXT: v_mov_b32_e32 v17, s6
-; GFX9-NEXT: .LBB109_5: ; %end
+; GFX9-NEXT: .LBB109_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v17, v31, v17, s4
; GFX9-NEXT: v_perm_b32 v3, v30, v3, s4
@@ -89381,10 +90503,8 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_perm_b32 v2, v44, v2, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_readlane_b32 s30, v63, 14
-; GFX9-NEXT: v_readlane_b32 s31, v63, 15
-; GFX9-NEXT: v_readlane_b32 s55, v63, 13
-; GFX9-NEXT: v_readlane_b32 s54, v63, 12
+; GFX9-NEXT: v_readlane_b32 s30, v63, 12
+; GFX9-NEXT: v_readlane_b32 s31, v63, 13
; GFX9-NEXT: v_readlane_b32 s53, v63, 11
; GFX9-NEXT: v_readlane_b32 s52, v63, 10
; GFX9-NEXT: v_readlane_b32 s51, v63, 9
@@ -89504,42 +90624,41 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b32 s78, s27, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s48, s27, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s39, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s88, s27, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s49, s26, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s48, s26, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s90, s26, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s75, s25, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s38, s25, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s37, s25, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s77, s25, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s39, s24, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s38, s24, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s89, s24, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s72, s23, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s36, s23, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s35, s23, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s23, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s37, s22, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s36, s22, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s79, s22, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s21, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s34, s21, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s31, s21, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s63, s21, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s35, s20, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s34, s20, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s76, s20, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s19, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s30, s19, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 vcc_hi, s19, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s19, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s31, s18, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s30, s18, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s18, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s47, s17, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s95, s17, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s17, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 vcc_hi, s16, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 vcc_lo, s16, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s16, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s3, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s93, s3, 16
@@ -89559,9 +90678,64 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-TRUE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB109_3
+; GFX11-TRUE16-NEXT: .LBB109_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $vcc_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr95
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr30
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $vcc_hi
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr34
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr31
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr36
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr35
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr38
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr37
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr48
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr39
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s4
@@ -89913,69 +91087,19 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB109_5
-; GFX11-TRUE16-NEXT: .LBB109_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $vcc_hi
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr95
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr31
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr30
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr35
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr34
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr37
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr36
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr39
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr28
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr38
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr49
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr48
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: s_branch .LBB109_2
-; GFX11-TRUE16-NEXT: .LBB109_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
+; GFX11-TRUE16-NEXT: .LBB109_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v52, s26 :: v_dual_mov_b32 v51, s27
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s49 :: v_dual_mov_b32 v53, s48
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s24 :: v_dual_mov_b32 v50, s39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s48 :: v_dual_mov_b32 v53, s39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s24 :: v_dual_mov_b32 v50, s38
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, s25 :: v_dual_mov_b32 v48, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s38 :: v_dual_mov_b32 v38, s37
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s23 :: v_dual_mov_b32 v37, s36
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s20 :: v_dual_mov_b32 v36, s35
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s21 :: v_dual_mov_b32 v33, s34
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s18 :: v_dual_mov_b32 v32, s31
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, s30
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s16 :: v_dual_mov_b32 v27, vcc_hi
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s37 :: v_dual_mov_b32 v38, s36
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s23 :: v_dual_mov_b32 v37, s35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s20 :: v_dual_mov_b32 v36, s34
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s21 :: v_dual_mov_b32 v33, s31
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s18 :: v_dual_mov_b32 v32, s30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v29, vcc_hi
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s16 :: v_dual_mov_b32 v27, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s17 :: v_dual_mov_b32 v22, s2
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s95 :: v_dual_mov_b32 v24, s94
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s3 :: v_dual_mov_b32 v21, s93
@@ -89997,7 +91121,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s14 :: v_dual_mov_b32 v10, s12
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s10 :: v_dual_mov_b32 v12, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s6 :: v_dual_mov_b32 v14, s4
-; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v19, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v1, 0xc0c0004
@@ -90064,14 +91188,13 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v20, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v7
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v22
-; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-TRUE16-NEXT: s_clause 0x3
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[11:14], off offset:16
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[16:19], off offset:32
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-TRUE16-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-TRUE16-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-TRUE16-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-TRUE16-NEXT: v_readlane_b32 s38, v40, 4
@@ -90098,42 +91221,41 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s38, 4
; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s39, 5
; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s48, 6
-; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s49, 7
-; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 8
-; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 9
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 7
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 8
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b32 s78, s27, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s48, s27, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s39, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s88, s27, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s49, s26, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s48, s26, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s90, s26, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s75, s25, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s38, s25, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s37, s25, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s77, s25, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s39, s24, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s38, s24, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s89, s24, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s72, s23, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s36, s23, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s35, s23, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s23, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s37, s22, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s36, s22, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s79, s22, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s21, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s34, s21, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s31, s21, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s63, s21, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s35, s20, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s34, s20, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s76, s20, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s19, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s30, s19, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 vcc_hi, s19, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s19, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s31, s18, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s30, s18, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s18, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s47, s17, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s95, s17, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s17, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 vcc_hi, s16, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 vcc_lo, s16, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s16, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s3, 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s93, s3, 16
@@ -90153,9 +91275,64 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b64 s[8:9], s[16:17], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-FAKE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB109_3
+; GFX11-FAKE16-NEXT: .LBB109_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $vcc_lo
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr95
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr30
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $vcc_hi
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr34
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr31
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr36
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr35
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr38
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr37
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr48
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr39
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s5, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff0000
; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s4
@@ -90512,69 +91689,19 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v96, 24, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB109_5
-; GFX11-FAKE16-NEXT: .LBB109_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $vcc_hi
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr95
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr31
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr30
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr35
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr34
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr37
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr36
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr39
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr28
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr38
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr49
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr48
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: s_branch .LBB109_2
-; GFX11-FAKE16-NEXT: .LBB109_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB109_6
+; GFX11-FAKE16-NEXT: .LBB109_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v50, s26 :: v_dual_mov_b32 v49, s27
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, s49 :: v_dual_mov_b32 v51, s48
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, s24 :: v_dual_mov_b32 v48, s39
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s25 :: v_dual_mov_b32 v39, s38
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s22 :: v_dual_mov_b32 v38, s37
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, s48 :: v_dual_mov_b32 v51, s39
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, s24 :: v_dual_mov_b32 v48, s38
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s25 :: v_dual_mov_b32 v39, s37
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s22 :: v_dual_mov_b32 v38, s36
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s23 :: v_dual_mov_b32 v34, s20
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s36 :: v_dual_mov_b32 v28, s21
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s35 :: v_dual_mov_b32 v30, s18
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s34 :: v_dual_mov_b32 v32, s31
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s35 :: v_dual_mov_b32 v28, s21
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s34 :: v_dual_mov_b32 v30, s18
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s31 :: v_dual_mov_b32 v32, s30
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s19 :: v_dual_mov_b32 v25, s16
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s30 :: v_dual_mov_b32 v27, vcc_hi
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, vcc_hi :: v_dual_mov_b32 v27, vcc_lo
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s17 :: v_dual_mov_b32 v20, s2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s95 :: v_dual_mov_b32 v22, s94
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v18, s3 :: v_dual_mov_b32 v19, s93
@@ -90596,7 +91723,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v8, s12
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, s10 :: v_dual_mov_b32 v10, s8
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s4
-; GFX11-FAKE16-NEXT: .LBB109_5: ; %end
+; GFX11-FAKE16-NEXT: .LBB109_6: ; %end
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v17, v12, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v15, v15, v96, 0xc0c0004
@@ -90663,14 +91790,13 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v19, v21
; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v22, v23
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v13, v20
-; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 8
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 7
; GFX11-FAKE16-NEXT: s_clause 0x3
; GFX11-FAKE16-NEXT: scratch_store_b128 v0, v[1:4], off
; GFX11-FAKE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:16
; GFX11-FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:32
; GFX11-FAKE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48
-; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 9
-; GFX11-FAKE16-NEXT: v_readlane_b32 s49, v40, 7
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 8
; GFX11-FAKE16-NEXT: v_readlane_b32 s48, v40, 6
; GFX11-FAKE16-NEXT: v_readlane_b32 s39, v40, 5
; GFX11-FAKE16-NEXT: v_readlane_b32 s38, v40, 4
@@ -93263,545 +94389,638 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr44 : SGPR spill to VGPR lane
-; SI-NEXT: v_readfirstlane_b32 s7, v1
-; SI-NEXT: v_writelane_b32 v44, s18, 0
-; SI-NEXT: v_writelane_b32 v44, s17, 1
-; SI-NEXT: v_writelane_b32 v44, s16, 2
-; SI-NEXT: v_writelane_b32 v44, s22, 3
-; SI-NEXT: v_writelane_b32 v44, s23, 4
-; SI-NEXT: v_writelane_b32 v44, s21, 5
-; SI-NEXT: v_writelane_b32 v44, s20, 6
-; SI-NEXT: v_writelane_b32 v44, s26, 7
-; SI-NEXT: v_writelane_b32 v44, s27, 8
-; SI-NEXT: v_writelane_b32 v44, s25, 9
-; SI-NEXT: v_writelane_b32 v44, s24, 10
-; SI-NEXT: v_writelane_b32 v44, s29, 11
-; SI-NEXT: v_writelane_b32 v44, s28, 12
; SI-NEXT: s_mov_b32 s6, s19
-; SI-NEXT: v_readfirstlane_b32 s89, v30
-; SI-NEXT: v_readfirstlane_b32 s63, v29
-; SI-NEXT: v_readfirstlane_b32 s72, v28
-; SI-NEXT: v_readfirstlane_b32 s75, v27
-; SI-NEXT: v_readfirstlane_b32 s77, v26
+; SI-NEXT: v_writelane_b32 v44, s18, 0
+; SI-NEXT: v_readfirstlane_b32 s34, v30
+; SI-NEXT: v_readfirstlane_b32 s72, v29
+; SI-NEXT: v_readfirstlane_b32 s73, v28
+; SI-NEXT: v_readfirstlane_b32 s76, v27
+; SI-NEXT: v_readfirstlane_b32 s78, v26
; SI-NEXT: v_readfirstlane_b32 s59, v25
; SI-NEXT: v_readfirstlane_b32 s60, v24
-; SI-NEXT: v_readfirstlane_b32 s62, v23
-; SI-NEXT: v_readfirstlane_b32 s73, v22
+; SI-NEXT: v_readfirstlane_b32 s63, v23
+; SI-NEXT: v_readfirstlane_b32 s75, v22
; SI-NEXT: v_readfirstlane_b32 s47, v21
; SI-NEXT: v_readfirstlane_b32 s56, v20
-; SI-NEXT: v_readfirstlane_b32 s58, v19
+; SI-NEXT: v_readfirstlane_b32 s62, v19
; SI-NEXT: v_readfirstlane_b32 s61, v18
; SI-NEXT: v_readfirstlane_b32 s43, v17
; SI-NEXT: v_readfirstlane_b32 s44, v16
-; SI-NEXT: v_readfirstlane_b32 s46, v15
+; SI-NEXT: v_readfirstlane_b32 s58, v15
; SI-NEXT: v_readfirstlane_b32 s57, v14
; SI-NEXT: v_readfirstlane_b32 s15, v13
; SI-NEXT: v_readfirstlane_b32 s40, v12
-; SI-NEXT: v_readfirstlane_b32 s42, v11
+; SI-NEXT: v_readfirstlane_b32 s46, v11
; SI-NEXT: v_readfirstlane_b32 s45, v10
; SI-NEXT: v_readfirstlane_b32 s11, v9
; SI-NEXT: v_readfirstlane_b32 s12, v8
-; SI-NEXT: v_readfirstlane_b32 s14, v7
+; SI-NEXT: v_readfirstlane_b32 s42, v7
; SI-NEXT: v_readfirstlane_b32 s41, v6
-; SI-NEXT: v_readfirstlane_b32 s19, v5
+; SI-NEXT: v_readfirstlane_b32 s8, v5
; SI-NEXT: v_readfirstlane_b32 s9, v4
-; SI-NEXT: v_readfirstlane_b32 s10, v3
+; SI-NEXT: v_readfirstlane_b32 s14, v3
; SI-NEXT: v_readfirstlane_b32 s13, v2
-; SI-NEXT: v_readfirstlane_b32 s90, v0
-; SI-NEXT: v_writelane_b32 v44, s7, 13
-; SI-NEXT: v_writelane_b32 v44, s90, 14
+; SI-NEXT: v_readfirstlane_b32 s10, v1
+; SI-NEXT: v_readfirstlane_b32 s19, v0
+; SI-NEXT: v_writelane_b32 v44, s6, 1
+; SI-NEXT: v_writelane_b32 v44, s63, 2
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_readfirstlane_b32 s87, v32
-; SI-NEXT: v_readfirstlane_b32 s86, v33
-; SI-NEXT: v_readfirstlane_b32 s98, v34
-; SI-NEXT: v_readfirstlane_b32 s99, v35
-; SI-NEXT: v_readfirstlane_b32 s82, v36
+; SI-NEXT: v_readfirstlane_b32 s84, v32
+; SI-NEXT: v_readfirstlane_b32 s83, v33
+; SI-NEXT: v_readfirstlane_b32 s96, v34
+; SI-NEXT: v_readfirstlane_b32 s97, v35
+; SI-NEXT: v_readfirstlane_b32 s80, v36
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_readfirstlane_b32 s81, v37
+; SI-NEXT: v_readfirstlane_b32 s71, v37
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_readfirstlane_b32 s84, v38
+; SI-NEXT: v_readfirstlane_b32 s82, v38
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_readfirstlane_b32 s85, v39
+; SI-NEXT: v_readfirstlane_b32 s87, v39
; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_readfirstlane_b32 s68, v48
+; SI-NEXT: v_readfirstlane_b32 s66, v48
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_readfirstlane_b32 s67, v49
+; SI-NEXT: v_readfirstlane_b32 s65, v49
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_readfirstlane_b32 s70, v50
+; SI-NEXT: v_readfirstlane_b32 s68, v50
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_readfirstlane_b32 s80, v51
+; SI-NEXT: v_readfirstlane_b32 s70, v51
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_readfirstlane_b32 s53, v52
+; SI-NEXT: v_readfirstlane_b32 s36, v52
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_readfirstlane_b32 s52, v53
+; SI-NEXT: v_readfirstlane_b32 s50, v53
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_readfirstlane_b32 s64, v54
+; SI-NEXT: v_readfirstlane_b32 s54, v54
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s66, v55
+; SI-NEXT: v_readfirstlane_b32 s64, v55
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_readfirstlane_b32 s48, v40
+; SI-NEXT: v_readfirstlane_b32 s37, v40
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_readfirstlane_b32 s39, v41
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s51, v42
-; SI-NEXT: s_cbranch_scc0 .LBB111_4
+; SI-NEXT: v_readfirstlane_b32 s49, v42
+; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s17, 24
-; SI-NEXT: s_or_b32 s74, s5, s4
-; SI-NEXT: s_and_b32 s4, s18, 0xff
+; SI-NEXT: s_or_b32 s98, s5, s4
+; SI-NEXT: v_readlane_b32 s4, v44, 0
+; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s6, 24
-; SI-NEXT: s_or_b32 s76, s5, s4
+; SI-NEXT: s_or_b32 s85, s5, s4
; SI-NEXT: s_and_b32 s4, s20, 0xff
; SI-NEXT: s_lshl_b32 s5, s21, 8
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s22, 0xff
-; SI-NEXT: s_mov_b32 s8, s6
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s23, 24
-; SI-NEXT: s_or_b32 s79, s6, s5
+; SI-NEXT: s_or_b32 s88, s6, s5
; SI-NEXT: s_and_b32 s5, s24, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s25, 24
-; SI-NEXT: s_or_b32 s78, s6, s5
+; SI-NEXT: s_or_b32 s79, s6, s5
; SI-NEXT: s_and_b32 s5, s26, 0xff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s27, 24
-; SI-NEXT: s_or_b32 s88, s6, s5
+; SI-NEXT: s_or_b32 s90, s6, s5
; SI-NEXT: s_and_b32 s5, s28, 0xff
; SI-NEXT: s_lshl_b32 s6, s29, 8
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_and_b32 s6, s90, 0xff
+; SI-NEXT: s_and_b32 s6, s19, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s7, 24
-; SI-NEXT: s_or_b32 s91, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s10, 24
+; SI-NEXT: s_or_b32 s92, s7, s6
; SI-NEXT: s_and_b32 s6, s13, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s10, 24
-; SI-NEXT: s_or_b32 s90, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s14, 24
+; SI-NEXT: s_or_b32 s91, s7, s6
; SI-NEXT: s_and_b32 s6, s9, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s19, 24
-; SI-NEXT: s_or_b32 s92, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_or_b32 s93, s7, s6
; SI-NEXT: s_and_b32 s6, s41, 0xff
-; SI-NEXT: s_lshl_b32 s7, s14, 8
-; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_and_b32 s7, s12, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s11, 24
-; SI-NEXT: s_or_b32 s94, s17, s7
-; SI-NEXT: s_and_b32 s7, s45, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s42, 24
-; SI-NEXT: s_or_b32 s93, s17, s7
-; SI-NEXT: s_and_b32 s7, s40, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s15, 24
-; SI-NEXT: s_or_b32 s95, s17, s7
-; SI-NEXT: s_and_b32 s7, s57, 0xff
-; SI-NEXT: s_lshl_b32 s17, s46, 8
-; SI-NEXT: s_or_b32 vcc_lo, s7, s17
-; SI-NEXT: s_and_b32 s7, s44, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s43, 24
-; SI-NEXT: s_or_b32 s31, s17, s7
-; SI-NEXT: s_and_b32 s7, s61, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s58, 24
-; SI-NEXT: s_or_b32 s30, s17, s7
-; SI-NEXT: s_and_b32 s7, s56, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s47, 24
-; SI-NEXT: s_or_b32 s34, s17, s7
-; SI-NEXT: s_and_b32 s7, s73, 0xff
-; SI-NEXT: s_lshl_b32 s17, s62, 8
-; SI-NEXT: s_or_b32 vcc_hi, s7, s17
-; SI-NEXT: s_and_b32 s7, s60, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s59, 24
-; SI-NEXT: s_or_b32 s36, s17, s7
-; SI-NEXT: s_and_b32 s7, s77, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s75, 24
-; SI-NEXT: s_or_b32 s35, s17, s7
-; SI-NEXT: s_and_b32 s7, s72, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s63, 24
-; SI-NEXT: s_or_b32 s37, s17, s7
-; SI-NEXT: s_and_b32 s7, s89, 0xff
-; SI-NEXT: s_lshl_b32 s17, s51, 8
-; SI-NEXT: s_or_b32 s71, s7, s17
-; SI-NEXT: s_and_b32 s7, s39, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s48, 24
-; SI-NEXT: s_or_b32 s49, s17, s7
-; SI-NEXT: s_and_b32 s7, s66, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s64, 24
-; SI-NEXT: s_or_b32 s38, s17, s7
-; SI-NEXT: s_and_b32 s7, s52, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s53, 24
-; SI-NEXT: s_or_b32 s50, s17, s7
-; SI-NEXT: s_and_b32 s7, s80, 0xff
-; SI-NEXT: s_lshl_b32 s17, s70, 8
-; SI-NEXT: s_or_b32 s83, s7, s17
-; SI-NEXT: s_and_b32 s7, s67, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s68, 24
-; SI-NEXT: s_or_b32 s55, s17, s7
-; SI-NEXT: s_and_b32 s7, s85, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s84, 24
-; SI-NEXT: s_or_b32 s54, s17, s7
-; SI-NEXT: s_and_b32 s7, s81, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s82, 24
-; SI-NEXT: s_or_b32 s65, s17, s7
-; SI-NEXT: s_and_b32 s7, s99, 0xff
-; SI-NEXT: s_lshl_b32 s17, s98, 8
-; SI-NEXT: s_or_b32 s16, s7, s17
-; SI-NEXT: s_and_b32 s7, s86, 0xff
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_lshl_b32 s17, s87, 24
-; SI-NEXT: s_or_b32 s69, s17, s7
-; SI-NEXT: s_lshl_b32 s17, s4, 16
-; SI-NEXT: s_lshl_b32 s18, s5, 16
-; SI-NEXT: s_lshl_b32 s7, s6, 16
-; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
-; SI-NEXT: s_lshl_b32 s97, vcc_hi, 16
-; SI-NEXT: s_lshl_b32 s96, s71, 16
-; SI-NEXT: s_lshl_b32 s83, s83, 16
-; SI-NEXT: s_lshl_b32 s71, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB111_3
-; SI-NEXT: .LBB111_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s99, s99, 3
-; SI-NEXT: s_and_b32 s4, s99, 0xff
-; SI-NEXT: s_lshl_b32 s5, s98, 8
-; SI-NEXT: s_add_i32 s86, s86, 3
+; SI-NEXT: s_lshl_b32 s7, s42, 8
+; SI-NEXT: s_or_b32 s69, s6, s7
+; SI-NEXT: s_and_b32 s6, s12, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s11, 24
+; SI-NEXT: s_or_b32 s95, s7, s6
+; SI-NEXT: s_and_b32 s6, s45, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s46, 24
+; SI-NEXT: s_or_b32 s94, s7, s6
+; SI-NEXT: s_and_b32 s6, s40, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s15, 24
+; SI-NEXT: s_or_b32 vcc_lo, s7, s6
+; SI-NEXT: s_and_b32 s6, s57, 0xff
+; SI-NEXT: s_lshl_b32 s7, s58, 8
+; SI-NEXT: s_or_b32 s81, s6, s7
+; SI-NEXT: s_and_b32 s6, s44, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s43, 24
+; SI-NEXT: s_or_b32 s30, s7, s6
+; SI-NEXT: s_and_b32 s6, s61, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s62, 24
+; SI-NEXT: s_or_b32 vcc_hi, s7, s6
+; SI-NEXT: s_and_b32 s6, s56, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s47, 24
+; SI-NEXT: s_or_b32 s31, s7, s6
+; SI-NEXT: s_and_b32 s6, s75, 0xff
+; SI-NEXT: s_lshl_b32 s7, s63, 8
+; SI-NEXT: s_or_b32 s86, s6, s7
+; SI-NEXT: s_and_b32 s6, s60, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s59, 24
+; SI-NEXT: s_or_b32 s35, s7, s6
+; SI-NEXT: s_and_b32 s6, s78, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s76, 24
+; SI-NEXT: s_or_b32 s99, s7, s6
+; SI-NEXT: s_and_b32 s6, s73, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s72, 24
+; SI-NEXT: s_or_b32 s38, s7, s6
+; SI-NEXT: s_and_b32 s6, s34, 0xff
+; SI-NEXT: s_lshl_b32 s7, s49, 8
+; SI-NEXT: s_or_b32 s18, s6, s7
+; SI-NEXT: s_and_b32 s6, s39, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s37, 24
+; SI-NEXT: s_or_b32 s48, s7, s6
+; SI-NEXT: s_and_b32 s6, s64, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s54, 24
+; SI-NEXT: s_or_b32 s89, s7, s6
+; SI-NEXT: s_and_b32 s6, s50, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s36, 24
+; SI-NEXT: s_or_b32 s51, s7, s6
+; SI-NEXT: s_and_b32 s6, s70, 0xff
+; SI-NEXT: s_lshl_b32 s7, s68, 8
+; SI-NEXT: s_mov_b32 s74, s60
+; SI-NEXT: s_mov_b32 s60, s57
+; SI-NEXT: s_mov_b32 s57, s58
+; SI-NEXT: s_mov_b32 s58, s44
+; SI-NEXT: s_mov_b32 s44, s41
+; SI-NEXT: s_mov_b32 s41, s42
+; SI-NEXT: s_mov_b32 s42, s12
+; SI-NEXT: s_mov_b32 s12, s9
+; SI-NEXT: s_mov_b32 s9, s10
+; SI-NEXT: s_mov_b32 s10, s28
+; SI-NEXT: s_mov_b32 s28, s24
+; SI-NEXT: s_mov_b32 s24, s26
+; SI-NEXT: s_mov_b32 s26, s23
+; SI-NEXT: s_mov_b32 s23, s17
+; SI-NEXT: s_or_b32 s17, s6, s7
+; SI-NEXT: s_and_b32 s6, s65, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s66, 24
+; SI-NEXT: s_or_b32 s53, s7, s6
+; SI-NEXT: s_and_b32 s6, s87, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s82, 24
+; SI-NEXT: s_or_b32 s52, s7, s6
+; SI-NEXT: s_and_b32 s6, s71, 0xff
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s80, 24
+; SI-NEXT: s_or_b32 s55, s7, s6
+; SI-NEXT: s_and_b32 s6, s97, 0xff
+; SI-NEXT: s_lshl_b32 s7, s96, 8
+; SI-NEXT: s_mov_b32 s63, s59
+; SI-NEXT: s_mov_b32 s59, s47
+; SI-NEXT: s_mov_b32 s47, s43
+; SI-NEXT: s_mov_b32 s43, s15
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s11, s19
+; SI-NEXT: s_mov_b32 s19, s29
+; SI-NEXT: s_mov_b32 s29, s27
+; SI-NEXT: s_mov_b32 s27, s21
+; SI-NEXT: s_mov_b32 s21, s16
+; SI-NEXT: s_or_b32 s16, s6, s7
+; SI-NEXT: s_and_b32 s6, s83, 0xff
+; SI-NEXT: s_mov_b32 s77, s76
+; SI-NEXT: s_mov_b32 s76, s75
+; SI-NEXT: s_mov_b32 s75, s73
+; SI-NEXT: s_mov_b32 s73, s72
+; SI-NEXT: s_mov_b32 s72, s61
+; SI-NEXT: s_mov_b32 s61, s62
+; SI-NEXT: s_mov_b32 s62, s56
+; SI-NEXT: s_mov_b32 s56, s45
+; SI-NEXT: s_mov_b32 s45, s46
+; SI-NEXT: s_mov_b32 s46, s40
+; SI-NEXT: s_mov_b32 s40, s13
+; SI-NEXT: s_mov_b32 s13, s14
+; SI-NEXT: s_mov_b32 s14, s8
+; SI-NEXT: s_mov_b32 s8, s25
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s7, s84, 24
+; SI-NEXT: s_or_b32 s67, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s4, 16
+; SI-NEXT: s_lshl_b32 s6, s5, 16
+; SI-NEXT: s_lshl_b32 s69, s69, 16
+; SI-NEXT: s_lshl_b32 s81, s81, 16
+; SI-NEXT: s_lshl_b32 s86, s86, 16
+; SI-NEXT: s_lshl_b32 s18, s18, 16
+; SI-NEXT: s_mov_b32 s25, s8
+; SI-NEXT: s_mov_b32 s8, s14
+; SI-NEXT: s_mov_b32 s14, s13
+; SI-NEXT: s_mov_b32 s13, s40
+; SI-NEXT: s_mov_b32 s40, s46
+; SI-NEXT: s_mov_b32 s46, s45
+; SI-NEXT: s_mov_b32 s45, s56
+; SI-NEXT: s_mov_b32 s56, s62
+; SI-NEXT: s_mov_b32 s62, s61
+; SI-NEXT: s_mov_b32 s61, s72
+; SI-NEXT: s_mov_b32 s72, s73
+; SI-NEXT: s_mov_b32 s73, s75
+; SI-NEXT: s_mov_b32 s75, s76
+; SI-NEXT: s_mov_b32 s76, s77
+; SI-NEXT: s_mov_b32 s77, s85
+; SI-NEXT: s_lshl_b32 s85, s17, 16
+; SI-NEXT: s_mov_b32 s17, s23
+; SI-NEXT: s_mov_b32 s23, s26
+; SI-NEXT: s_mov_b32 s26, s24
+; SI-NEXT: s_mov_b32 s24, s28
+; SI-NEXT: s_mov_b32 s28, s10
+; SI-NEXT: s_mov_b32 s10, s9
+; SI-NEXT: s_mov_b32 s9, s12
+; SI-NEXT: s_mov_b32 s12, s42
+; SI-NEXT: s_mov_b32 s42, s41
+; SI-NEXT: s_mov_b32 s41, s44
+; SI-NEXT: s_mov_b32 s44, s58
+; SI-NEXT: s_mov_b32 s58, s57
+; SI-NEXT: s_mov_b32 s57, s60
+; SI-NEXT: s_mov_b32 s60, s74
+; SI-NEXT: s_mov_b32 s74, s98
+; SI-NEXT: s_lshl_b32 s98, s16, 16
+; SI-NEXT: s_mov_b32 s16, s21
+; SI-NEXT: s_mov_b32 s21, s27
+; SI-NEXT: s_mov_b32 s27, s29
+; SI-NEXT: s_mov_b32 s29, s19
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_mov_b32 s11, s15
+; SI-NEXT: s_mov_b32 s15, s43
+; SI-NEXT: s_mov_b32 s43, s47
+; SI-NEXT: s_mov_b32 s47, s59
+; SI-NEXT: s_mov_b32 s59, s63
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB111_3
+; SI-NEXT: .LBB111_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr69
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr81
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr86
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr99
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr18
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr85
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr98
+; SI-NEXT: ; implicit-def: $sgpr67
+; SI-NEXT: .LBB111_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB111_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s97, s97, 3
+; SI-NEXT: s_and_b32 s4, s97, 0xff
+; SI-NEXT: s_lshl_b32 s5, s96, 8
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s6, s86, 0xff
+; SI-NEXT: s_lshl_b32 s5, s84, 24
+; SI-NEXT: s_add_i32 s84, s83, 3
+; SI-NEXT: s_and_b32 s6, s84, 0xff
; SI-NEXT: s_addk_i32 s4, 0x300
-; SI-NEXT: s_lshl_b32 s5, s87, 24
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s85, s85, 3
+; SI-NEXT: s_add_i32 s83, s87, 3
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s85, 0xff
-; SI-NEXT: s_lshl_b32 s6, s84, 8
-; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_and_b32 s5, s83, 0xff
+; SI-NEXT: s_lshl_b32 s6, s82, 8
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s7, s81, 0xff
+; SI-NEXT: s_and_b32 s7, s71, 0xff
; SI-NEXT: s_addk_i32 s5, 0x300
-; SI-NEXT: s_lshl_b32 s6, s82, 24
+; SI-NEXT: s_lshl_b32 s6, s80, 24
; SI-NEXT: s_lshl_b32 s7, s7, 16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s80, 0xff
-; SI-NEXT: s_lshl_b32 s7, s70, 8
-; SI-NEXT: s_add_i32 s67, s67, 3
+; SI-NEXT: s_and_b32 s6, s70, 0xff
+; SI-NEXT: s_lshl_b32 s7, s68, 8
+; SI-NEXT: s_add_i32 s65, s65, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s67, 0xff
+; SI-NEXT: s_and_b32 s74, s65, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s68, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s7, s66, 24
+; SI-NEXT: s_lshl_b32 s74, s74, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
-; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: s_or_b32 vcc_lo, s7, s6
-; SI-NEXT: s_and_b32 s6, s66, 0xff
-; SI-NEXT: s_lshl_b32 s7, s64, 8
-; SI-NEXT: s_add_i32 s52, s52, 3
+; SI-NEXT: s_or_b32 s7, s7, s74
+; SI-NEXT: s_add_i32 s64, s64, 3
+; SI-NEXT: s_or_b32 s52, s7, s6
+; SI-NEXT: s_and_b32 s6, s64, 0xff
+; SI-NEXT: s_lshl_b32 s7, s54, 8
+; SI-NEXT: s_add_i32 s50, s50, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s52, 0xff
+; SI-NEXT: s_and_b32 s74, s50, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s53, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s7, s36, 24
+; SI-NEXT: s_lshl_b32 s74, s74, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s7, s7, s74
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s89, s89, 3
-; SI-NEXT: s_add_i32 vcc_hi, s6, 0x3000000
+; SI-NEXT: s_add_i32 s89, s34, 3
+; SI-NEXT: s_add_i32 s36, s6, 0x3000000
; SI-NEXT: s_and_b32 s6, s89, 0xff
-; SI-NEXT: s_lshl_b32 s7, s51, 8
-; SI-NEXT: s_add_i32 s39, s39, 3
+; SI-NEXT: s_lshl_b32 s7, s49, 8
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s39, 0xff
+; SI-NEXT: s_lshl_b32 s7, s37, 24
+; SI-NEXT: s_add_i32 s37, s39, 3
+; SI-NEXT: s_and_b32 s74, s37, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s48, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s74, s74, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s7, s7, s74
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s77, s77, 3
+; SI-NEXT: s_add_i32 s77, s78, 3
; SI-NEXT: s_add_i32 s89, s6, 0x3000000
; SI-NEXT: s_and_b32 s6, s77, 0xff
-; SI-NEXT: s_lshl_b32 s7, s75, 8
-; SI-NEXT: s_add_i32 s72, s72, 3
+; SI-NEXT: s_lshl_b32 s7, s76, 8
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s72, 0xff
+; SI-NEXT: s_lshl_b32 s7, s72, 24
+; SI-NEXT: s_add_i32 s72, s73, 3
+; SI-NEXT: s_and_b32 s63, s72, 0xff
; SI-NEXT: s_addk_i32 s6, 0x300
-; SI-NEXT: s_lshl_b32 s7, s63, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_lshl_b32 s63, s63, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s7, s7, s63
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_add_i32 s73, s73, 3
+; SI-NEXT: s_add_i32 s73, s75, 3
+; SI-NEXT: v_readlane_b32 s7, v44, 2
; SI-NEXT: s_add_i32 s63, s6, 0x3000000
; SI-NEXT: s_and_b32 s6, s73, 0xff
-; SI-NEXT: s_lshl_b32 s7, s62, 8
+; SI-NEXT: s_lshl_b32 s7, s7, 8
; SI-NEXT: s_add_i32 s60, s60, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s60, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s59, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_and_b32 s59, s60, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s59, s59, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s7, s7, s59
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s61, s61, 3
; SI-NEXT: s_add_i32 s59, s6, 0x3000000
; SI-NEXT: s_and_b32 s6, s61, 0xff
-; SI-NEXT: s_lshl_b32 s7, s58, 8
+; SI-NEXT: s_lshl_b32 s7, s62, 8
; SI-NEXT: s_add_i32 s56, s56, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s56, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s47, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_and_b32 s47, s56, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s47, s47, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
+; SI-NEXT: s_or_b32 s7, s7, s47
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s57, s57, 3
; SI-NEXT: s_add_i32 s47, s6, 0x3000000
; SI-NEXT: s_and_b32 s6, s57, 0xff
-; SI-NEXT: s_lshl_b32 s7, s46, 8
+; SI-NEXT: s_lshl_b32 s7, s58, 8
; SI-NEXT: s_add_i32 s44, s44, 3
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s17, s44, 0xff
-; SI-NEXT: s_addk_i32 s6, 0x300
; SI-NEXT: s_lshl_b32 s7, s43, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
+; SI-NEXT: s_and_b32 s43, s44, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s43, s43, 16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_or_b32 s7, s7, s17
-; SI-NEXT: s_add_i32 s45, s45, 3
+; SI-NEXT: s_or_b32 s7, s7, s43
; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s45, 0xff
-; SI-NEXT: s_lshl_b32 s17, s42, 8
+; SI-NEXT: s_add_i32 s45, s45, 3
+; SI-NEXT: s_add_i32 s43, s6, 0x3000000
+; SI-NEXT: s_and_b32 s6, s45, 0xff
+; SI-NEXT: s_lshl_b32 s7, s46, 8
; SI-NEXT: s_add_i32 s40, s40, 3
-; SI-NEXT: s_or_b32 s7, s17, s7
-; SI-NEXT: s_and_b32 s17, s40, 0xff
-; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s15, s15, 24
-; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s15, s15, s17
-; SI-NEXT: s_or_b32 s7, s15, s7
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s15, 24
+; SI-NEXT: s_and_b32 s15, s40, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s15, s15, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s15
+; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_add_i32 s41, s41, 3
-; SI-NEXT: s_add_i32 s15, s7, 0x3000000
-; SI-NEXT: s_and_b32 s7, s41, 0xff
-; SI-NEXT: s_lshl_b32 s14, s14, 8
+; SI-NEXT: s_add_i32 s15, s6, 0x3000000
+; SI-NEXT: s_and_b32 s6, s41, 0xff
+; SI-NEXT: s_lshl_b32 s7, s42, 8
; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: s_or_b32 s7, s14, s7
-; SI-NEXT: s_and_b32 s12, s12, 0xff
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s11, 24
+; SI-NEXT: s_and_b32 s11, s12, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s11, s11, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s11
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s13, s13, 3
+; SI-NEXT: s_add_i32 s11, s6, 0x3000000
+; SI-NEXT: s_and_b32 s6, s13, 0xff
+; SI-NEXT: s_lshl_b32 s7, s14, 8
+; SI-NEXT: s_add_i32 s9, s9, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_and_b32 s8, s9, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s8
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: s_add_i32 s8, s6, 0x3000000
+; SI-NEXT: s_and_b32 s6, s28, 0xff
+; SI-NEXT: s_lshl_b32 s7, s29, 8
+; SI-NEXT: s_add_i32 s19, s19, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s9, s19, 0xff
+; SI-NEXT: s_addk_i32 s6, 0x300
+; SI-NEXT: s_lshl_b32 s7, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 16
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_or_b32 s7, s7, s9
+; SI-NEXT: s_add_i32 s24, s24, 3
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s24, 0xff
+; SI-NEXT: s_lshl_b32 s9, s25, 8
+; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: s_or_b32 s7, s9, s7
+; SI-NEXT: s_and_b32 s10, s26, 0xff
; SI-NEXT: s_addk_i32 s7, 0x300
-; SI-NEXT: s_lshl_b32 s11, s11, 24
+; SI-NEXT: s_lshl_b32 s9, s27, 24
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_or_b32 s9, s9, s10
+; SI-NEXT: s_or_b32 s7, s9, s7
+; SI-NEXT: s_add_i32 s20, s20, 3
+; SI-NEXT: s_add_i32 s9, s7, 0x3000000
+; SI-NEXT: s_and_b32 s7, s20, 0xff
+; SI-NEXT: s_lshl_b32 s10, s21, 8
+; SI-NEXT: s_add_i32 s22, s22, 3
+; SI-NEXT: s_or_b32 s7, s10, s7
+; SI-NEXT: s_and_b32 s12, s22, 0xff
+; SI-NEXT: s_addk_i32 s7, 0x300
+; SI-NEXT: s_lshl_b32 s10, s23, 24
; SI-NEXT: s_lshl_b32 s12, s12, 16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s12
+; SI-NEXT: s_or_b32 s10, s10, s12
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s13, v44, 0
+; SI-NEXT: s_or_b32 s7, s10, s7
+; SI-NEXT: s_and_b32 s10, s16, 0xff
+; SI-NEXT: s_lshl_b32 s12, s17, 8
; SI-NEXT: s_add_i32 s13, s13, 3
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_and_b32 s11, s13, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
-; SI-NEXT: s_add_i32 s9, s9, 3
-; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_and_b32 s9, s9, 0xff
-; SI-NEXT: s_mov_b32 s16, s8
-; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s8, s19, 24
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: v_readlane_b32 s9, v44, 12
-; SI-NEXT: s_or_b32 s8, s8, s10
-; SI-NEXT: s_add_i32 s28, s9, 3
-; SI-NEXT: v_readlane_b32 s10, v44, 11
-; SI-NEXT: v_readlane_b32 s11, v44, 14
-; SI-NEXT: s_and_b32 s9, s28, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
-; SI-NEXT: s_add_i32 s19, s11, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: v_readlane_b32 s10, v44, 13
-; SI-NEXT: s_and_b32 s11, s19, 0xff
-; SI-NEXT: s_addk_i32 s9, 0x300
-; SI-NEXT: s_lshl_b32 s10, s10, 24
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: v_readlane_b32 s10, v44, 10
-; SI-NEXT: s_add_i32 s24, s10, 3
-; SI-NEXT: v_readlane_b32 s11, v44, 9
-; SI-NEXT: v_readlane_b32 s12, v44, 7
-; SI-NEXT: s_and_b32 s10, s24, 0xff
-; SI-NEXT: s_lshl_b32 s11, s11, 8
-; SI-NEXT: s_add_i32 s26, s12, 3
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v44, 8
-; SI-NEXT: s_and_b32 s12, s26, 0xff
+; SI-NEXT: s_or_b32 s10, s12, s10
+; SI-NEXT: v_readlane_b32 s12, v44, 1
+; SI-NEXT: s_and_b32 s13, s13, 0xff
; SI-NEXT: s_addk_i32 s10, 0x300
-; SI-NEXT: s_lshl_b32 s11, s11, 24
-; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_or_b32 s11, s11, s12
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: v_readlane_b32 s11, v44, 6
-; SI-NEXT: s_add_i32 s20, s11, 3
-; SI-NEXT: v_readlane_b32 s12, v44, 5
-; SI-NEXT: v_readlane_b32 s13, v44, 3
-; SI-NEXT: s_and_b32 s11, s20, 0xff
-; SI-NEXT: s_lshl_b32 s12, s12, 8
-; SI-NEXT: s_add_i32 s22, s13, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v44, 4
-; SI-NEXT: s_and_b32 s13, s22, 0xff
-; SI-NEXT: s_addk_i32 s11, 0x300
; SI-NEXT: s_lshl_b32 s12, s12, 24
; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s12, s13
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: v_readlane_b32 s12, v44, 2
-; SI-NEXT: s_add_i32 s12, s12, 3
-; SI-NEXT: v_readlane_b32 s13, v44, 1
-; SI-NEXT: v_readlane_b32 s14, v44, 0
-; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: s_lshl_b32 s13, s13, 8
-; SI-NEXT: s_add_i32 s14, s14, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: s_addk_i32 s12, 0x300
-; SI-NEXT: s_lshl_b32 s13, s16, 24
-; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_or_b32 s13, s13, s14
-; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_or_b32 s10, s12, s10
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 0x3000000
+; SI-NEXT: s_add_i32 s52, s52, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
; SI-NEXT: s_add_i32 s7, s7, 0x3000000
-; SI-NEXT: s_add_i32 s8, s8, 0x3000000
-; SI-NEXT: s_add_i32 s9, s9, 0x3000000
; SI-NEXT: s_add_i32 s10, s10, 0x3000000
-; SI-NEXT: s_add_i32 s11, s11, 0x3000000
-; SI-NEXT: s_add_i32 s12, s12, 0x3000000
-; SI-NEXT: s_and_b32 s76, s12, 0xffff0000
-; SI-NEXT: s_lshl_b32 s74, s12, 16
-; SI-NEXT: s_and_b32 s79, s11, 0xffff0000
-; SI-NEXT: s_lshl_b32 s17, s11, 16
-; SI-NEXT: s_and_b32 s88, s10, 0xffff0000
-; SI-NEXT: s_lshl_b32 s78, s10, 16
-; SI-NEXT: s_and_b32 s91, s9, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s9, 16
-; SI-NEXT: s_and_b32 s92, s8, 0xffff0000
-; SI-NEXT: s_lshl_b32 s90, s8, 16
-; SI-NEXT: s_and_b32 s94, s7, 0xffff0000
+; SI-NEXT: s_and_b32 s77, s10, 0xffff0000
+; SI-NEXT: s_lshl_b32 s74, s10, 16
+; SI-NEXT: s_and_b32 s88, s7, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_and_b32 s95, s15, 0xffff0000
-; SI-NEXT: s_lshl_b32 s93, s15, 16
-; SI-NEXT: s_and_b32 s31, s6, 0xffff0000
+; SI-NEXT: s_and_b32 s90, s9, 0xffff0000
+; SI-NEXT: s_lshl_b32 s79, s9, 16
+; SI-NEXT: s_and_b32 s92, s6, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s34, s47, 0xffff0000
-; SI-NEXT: s_lshl_b32 s30, s47, 16
-; SI-NEXT: s_and_b32 s36, s59, 0xffff0000
-; SI-NEXT: s_lshl_b32 s97, s59, 16
-; SI-NEXT: s_and_b32 s37, s63, 0xffff0000
-; SI-NEXT: s_lshl_b32 s35, s63, 16
-; SI-NEXT: s_and_b32 s49, s89, 0xffff0000
-; SI-NEXT: s_lshl_b32 s96, s89, 16
-; SI-NEXT: s_and_b32 s50, vcc_hi, 0xffff0000
-; SI-NEXT: s_lshl_b32 s38, vcc_hi, 16
-; SI-NEXT: s_and_b32 s55, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 s83, vcc_lo, 16
-; SI-NEXT: s_and_b32 s65, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s54, s5, 16
-; SI-NEXT: s_and_b32 s69, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s71, s4, 16
-; SI-NEXT: .LBB111_3: ; %end
+; SI-NEXT: s_and_b32 s93, s8, 0xffff0000
+; SI-NEXT: s_lshl_b32 s91, s8, 16
+; SI-NEXT: s_and_b32 s95, s11, 0xffff0000
+; SI-NEXT: s_lshl_b32 s69, s11, 16
+; SI-NEXT: s_and_b32 vcc_lo, s15, 0xffff0000
+; SI-NEXT: s_lshl_b32 s94, s15, 16
+; SI-NEXT: s_and_b32 s30, s43, 0xffff0000
+; SI-NEXT: s_lshl_b32 s81, s43, 16
+; SI-NEXT: s_and_b32 s31, s47, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_hi, s47, 16
+; SI-NEXT: s_and_b32 s35, s59, 0xffff0000
+; SI-NEXT: s_lshl_b32 s86, s59, 16
+; SI-NEXT: s_and_b32 s38, s63, 0xffff0000
+; SI-NEXT: s_lshl_b32 s99, s63, 16
+; SI-NEXT: s_and_b32 s48, s89, 0xffff0000
+; SI-NEXT: s_lshl_b32 s18, s89, 16
+; SI-NEXT: s_and_b32 s51, s36, 0xffff0000
+; SI-NEXT: s_lshl_b32 s89, s36, 16
+; SI-NEXT: s_and_b32 s53, s52, 0xffff0000
+; SI-NEXT: s_lshl_b32 s85, s52, 16
+; SI-NEXT: s_and_b32 s55, s5, 0xffff0000
+; SI-NEXT: s_lshl_b32 s52, s5, 16
+; SI-NEXT: s_and_b32 s67, s4, 0xffff0000
+; SI-NEXT: s_lshl_b32 s98, s4, 16
+; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s74
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s79
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s88
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_mul_f32_e64 v1, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s88
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s90
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v2, 1.0, s79
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s92
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s6
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s92
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s93
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s91
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s95
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s69
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, vcc_lo
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s93
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s94
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s31
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s30
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s81
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s34
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s31
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, vcc_hi
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s36
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s35
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v9
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s97
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s86
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s38
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s99
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s49
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s48
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v11
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s96
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s18
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s50
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s51
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s38
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s89
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s55
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s53
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s83
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s85
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s65
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s55
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s54
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s52
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s69
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s67
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v15
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s71
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s98
; SI-NEXT: v_readlane_b32 s30, v43, 34
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
; SI-NEXT: v_readlane_b32 s31, v43, 35
@@ -93845,41 +95064,6 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB111_4:
-; SI-NEXT: s_mov_b32 s8, s6
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr17
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr97
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr96
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr83
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr65
-; SI-NEXT: ; implicit-def: $sgpr71
-; SI-NEXT: ; implicit-def: $sgpr69
-; SI-NEXT: s_branch .LBB111_2
;
; VI-LABEL: bitcast_v64i8_to_v32bf16_scalar:
; VI: ; %bb.0:
@@ -93938,7 +95122,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-NEXT: s_cbranch_scc0 .LBB111_4
+; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -94030,8 +95214,17 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: v_perm_b32 v35, v33, v28, s4
; VI-NEXT: v_or_b32_e32 v15, v35, v15
-; VI-NEXT: s_cbranch_execnz .LBB111_3
-; VI-NEXT: .LBB111_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB111_3
+; VI-NEXT: .LBB111_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; VI-NEXT: .LBB111_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB111_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_u32_e32 v0, vcc, 3, v34
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v8, v0, v31, s4
@@ -94191,12 +95384,9 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; VI-NEXT: v_add_u32_e32 v13, vcc, 0x3000000, v13
; VI-NEXT: v_add_u32_e32 v14, vcc, 0x3000000, v14
; VI-NEXT: v_add_u32_e32 v15, vcc, 0x3000000, v15
-; VI-NEXT: .LBB111_3: ; %end
+; VI-NEXT: .LBB111_5: ; %end
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB111_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; VI-NEXT: s_branch .LBB111_2
;
; GFX9-LABEL: bitcast_v64i8_to_v32bf16_scalar:
; GFX9: ; %bb.0:
@@ -94255,7 +95445,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v8, 0xc0c0004
@@ -94361,8 +95551,17 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s5, s77
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_mov_b32_e32 v10, s5
-; GFX9-NEXT: s_cbranch_execnz .LBB111_3
-; GFX9-NEXT: .LBB111_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB111_3
+; GFX9-NEXT: .LBB111_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX9-NEXT: .LBB111_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_add_u32_e32 v0, 3, v34
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
@@ -94519,12 +95718,9 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v10, v16, 16, v10
; GFX9-NEXT: v_lshl_or_b32 v14, v27, 16, v14
; GFX9-NEXT: v_lshl_or_b32 v15, v25, 16, v15
-; GFX9-NEXT: .LBB111_3: ; %end
+; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB111_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX9-NEXT: s_branch .LBB111_2
;
; GFX11-TRUE16-LABEL: bitcast_v64i8_to_v32bf16_scalar:
; GFX11-TRUE16: ; %bb.0:
@@ -94581,7 +95777,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, 0xc0c0004 :: v_dual_lshlrev_b32 v13, 8, v50
@@ -94660,9 +95856,17 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s77
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v18.l
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-TRUE16-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB111_3
+; GFX11-TRUE16-NEXT: .LBB111_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s75, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -94779,12 +95983,9 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: .LBB111_3: ; %end
+; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB111_4:
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB111_2
;
; GFX11-FAKE16-LABEL: bitcast_v64i8_to_v32bf16_scalar:
; GFX11-FAKE16: ; %bb.0:
@@ -94841,7 +96042,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, 0xc0c0004
; GFX11-FAKE16-NEXT: s_and_b32 s76, s58, 0xff
@@ -94934,9 +96135,17 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v16, 16, v17
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, s76
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, s77
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s75
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-FAKE16-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB111_3
+; GFX11-FAKE16-NEXT: .LBB111_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s75, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s75, s75, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s75, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -95069,12 +96278,9 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v16, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v14, 16, v21
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v22
-; GFX11-FAKE16-NEXT: .LBB111_3: ; %end
+; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB111_4:
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB111_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
index 9a74ac7e2b943..37072b082797b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll
@@ -158,10 +158,18 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -180,7 +188,7 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -200,8 +208,6 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v18i32_to_v18f32_scalar:
; VI: ; %bb.0:
@@ -212,10 +218,18 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -234,7 +248,7 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -254,8 +268,6 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v18i32_to_v18f32_scalar:
; GFX9: ; %bb.0:
@@ -266,10 +278,18 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -288,7 +308,7 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -308,8 +328,6 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v18i32_to_v18f32_scalar:
; GFX11: ; %bb.0:
@@ -317,11 +335,16 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
; GFX11-NEXT: s_add_i32 s27, s27, 3
@@ -340,7 +363,7 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -352,8 +375,6 @@ define inreg <18 x float> @bitcast_v18i32_to_v18f32_scalar(<18 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -542,10 +563,18 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
; SI-NEXT: v_add_f32_e64 v16, s52, 1.0
; SI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -564,10 +593,8 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -600,7 +627,7 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -653,10 +680,18 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
; VI-NEXT: v_add_f32_e64 v16, s52, 1.0
; VI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -675,10 +710,8 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -711,7 +744,7 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -764,10 +797,18 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX9-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -786,10 +827,8 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -822,7 +861,7 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -876,11 +915,16 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX11-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -900,8 +944,6 @@ define inreg <18 x i32> @bitcast_v18f32_to_v18i32_scalar(<18 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1104,10 +1146,18 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1126,7 +1176,7 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1146,8 +1196,6 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v18i32_to_v9i64_scalar:
; VI: ; %bb.0:
@@ -1158,10 +1206,18 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1180,7 +1236,7 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1200,8 +1256,6 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v18i32_to_v9i64_scalar:
; GFX9: ; %bb.0:
@@ -1212,10 +1266,18 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1234,7 +1296,7 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1254,8 +1316,6 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v18i32_to_v9i64_scalar:
; GFX11: ; %bb.0:
@@ -1263,11 +1323,16 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
; GFX11-NEXT: s_add_i32 s27, s27, 3
@@ -1286,7 +1351,7 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1298,8 +1363,6 @@ define inreg <9 x i64> @bitcast_v18i32_to_v9i64_scalar(<18 x i32> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1474,10 +1537,18 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1496,7 +1567,7 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1516,8 +1587,6 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v9i64_to_v18i32_scalar:
; VI: ; %bb.0:
@@ -1528,10 +1597,18 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -1550,7 +1627,7 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1570,8 +1647,6 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v9i64_to_v18i32_scalar:
; GFX9: ; %bb.0:
@@ -1582,10 +1657,18 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -1604,7 +1687,7 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1624,8 +1707,6 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v9i64_to_v18i32_scalar:
; GFX11: ; %bb.0:
@@ -1633,11 +1714,16 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
; GFX11-NEXT: s_add_u32 s26, s26, 3
@@ -1656,7 +1742,7 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1668,8 +1754,6 @@ define inreg <18 x i32> @bitcast_v9i64_to_v18i32_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1839,10 +1923,18 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1861,7 +1953,7 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1881,8 +1973,6 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v18i32_to_v9f64_scalar:
; VI: ; %bb.0:
@@ -1893,10 +1983,18 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1915,7 +2013,7 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1935,8 +2033,6 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v18i32_to_v9f64_scalar:
; GFX9: ; %bb.0:
@@ -1947,10 +2043,18 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1969,7 +2073,7 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1989,8 +2093,6 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v18i32_to_v9f64_scalar:
; GFX11: ; %bb.0:
@@ -1998,11 +2100,16 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
; GFX11-NEXT: s_add_i32 s27, s27, 3
@@ -2021,7 +2128,7 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2033,8 +2140,6 @@ define inreg <9 x double> @bitcast_v18i32_to_v9f64_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2196,10 +2301,18 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -2209,10 +2322,8 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2245,7 +2356,7 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -2298,10 +2409,18 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -2311,10 +2430,8 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -2347,7 +2464,7 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -2400,10 +2517,18 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -2413,10 +2538,8 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -2449,7 +2572,7 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -2503,11 +2626,16 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -2518,8 +2646,6 @@ define inreg <18 x i32> @bitcast_v9f64_to_v18i32_scalar(<9 x double> inreg %a, i
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -3110,7 +3236,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s60, s5, 16
; SI-NEXT: s_lshr_b32 s61, s7, 16
@@ -3130,8 +3256,34 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3168,7 +3320,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s74, s21, 16
; SI-NEXT: s_lshr_b32 s75, s19, 16
; SI-NEXT: s_lshr_b32 s76, s17, 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s9, s56, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff
; SI-NEXT: s_or_b32 s9, s11, s9
@@ -3242,26 +3394,6 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v16, s4
; SI-NEXT: v_mov_b32_e32 v17, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v18i32_to_v36i16_scalar:
; VI: ; %bb.0:
@@ -3272,7 +3404,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -3292,8 +3424,34 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -3330,7 +3488,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -3404,26 +3562,6 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v18i32_to_v36i16_scalar:
; GFX9: ; %bb.0:
@@ -3434,7 +3572,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -3454,8 +3592,34 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -3492,7 +3656,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s58
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s57
@@ -3530,26 +3694,6 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v18i32_to_v36i16_scalar:
; GFX11: ; %bb.0:
@@ -3557,7 +3701,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_mov_b32 s46, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-NEXT: s_lshr_b32 s5, s28, 16
@@ -3577,9 +3721,34 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
; GFX11-NEXT: s_add_i32 s27, s27, 3
@@ -3616,7 +3785,7 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s45
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s44
@@ -3646,26 +3815,6 @@ define inreg <36 x i16> @bitcast_v18i32_to_v36i16_scalar(<18 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, s6
; GFX11-NEXT: v_dual_mov_b32 v16, s5 :: v_dual_mov_b32 v17, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4472,7 +4621,7 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s15, s72, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -4528,8 +4677,17 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -4620,7 +4778,7 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -4660,9 +4818,6 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v36i16_to_v18i32_scalar:
; VI: ; %bb.0:
@@ -4710,7 +4865,7 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s15, s72, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -4766,8 +4921,17 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -4858,7 +5022,7 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -4898,9 +5062,6 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v36i16_to_v18i32_scalar:
; GFX9: ; %bb.0:
@@ -4960,10 +5121,18 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -4982,10 +5151,8 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -5018,7 +5185,7 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -5077,11 +5244,16 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -5101,8 +5273,6 @@ define inreg <18 x i32> @bitcast_v36i16_to_v18i32_scalar(<36 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v16, s16, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v17, s17, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5678,7 +5848,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s60, s5, 16
; SI-NEXT: s_lshr_b32 s61, s7, 16
@@ -5698,8 +5868,34 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -5736,7 +5932,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s74, s21, 16
; SI-NEXT: s_lshr_b32 s75, s19, 16
; SI-NEXT: s_lshr_b32 s76, s17, 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s9, s56, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff
; SI-NEXT: s_or_b32 s9, s11, s9
@@ -5810,26 +6006,6 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v16, s4
; SI-NEXT: v_mov_b32_e32 v17, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v18i32_to_v36f16_scalar:
; VI: ; %bb.0:
@@ -5840,7 +6016,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -5860,8 +6036,34 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -5898,7 +6100,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -5972,26 +6174,6 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v18i32_to_v36f16_scalar:
; GFX9: ; %bb.0:
@@ -6002,7 +6184,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -6022,8 +6204,34 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -6060,7 +6268,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s58
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s57
@@ -6098,26 +6306,6 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v18i32_to_v36f16_scalar:
; GFX11: ; %bb.0:
@@ -6125,7 +6313,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_mov_b32 s46, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-NEXT: s_lshr_b32 s5, s28, 16
@@ -6145,9 +6333,34 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s29, s29, 3
; GFX11-NEXT: s_add_i32 s28, s28, 3
; GFX11-NEXT: s_add_i32 s27, s27, 3
@@ -6184,7 +6397,7 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s45
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s44
@@ -6214,26 +6427,6 @@ define inreg <36 x half> @bitcast_v18i32_to_v36f16_scalar(<18 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, s6
; GFX11-NEXT: v_dual_mov_b32 v16, s5 :: v_dual_mov_b32 v17, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7114,7 +7307,7 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s15, s13, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -7170,8 +7363,17 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s91
; SI-NEXT: v_cvt_f32_f16_e32 v2, s90
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
@@ -7316,11 +7518,8 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v18, v17
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -7353,7 +7552,7 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
+; SI-NEXT: .LBB19_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -7422,7 +7621,7 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s15, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -7478,8 +7677,17 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v17, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s91
; VI-NEXT: v_mov_b32_e32 v2, s90
@@ -7553,11 +7761,8 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v18, v18, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v17, s6, v17
; VI-NEXT: v_or_b32_e32 v17, v17, v18
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -7590,7 +7795,7 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
+; VI-NEXT: .LBB19_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -7671,10 +7876,18 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v17, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v17 op_sel_hi:[1,0]
@@ -7694,10 +7907,8 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v15, s51, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v17 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -7730,7 +7941,7 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -7789,11 +8000,16 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -7813,8 +8029,6 @@ define inreg <18 x i32> @bitcast_v36f16_to_v18i32_scalar(<36 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s16 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s17 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8021,10 +8235,18 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
; SI-NEXT: v_add_f32_e64 v16, s52, 1.0
; SI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -8043,10 +8265,8 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -8079,7 +8299,7 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -8132,10 +8352,18 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
; VI-NEXT: v_add_f32_e64 v16, s52, 1.0
; VI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -8154,10 +8382,8 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -8190,7 +8416,7 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -8243,10 +8469,18 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX9-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -8265,10 +8499,8 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -8301,7 +8533,7 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -8355,11 +8587,16 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX11-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -8379,8 +8616,6 @@ define inreg <9 x i64> @bitcast_v18f32_to_v9i64_scalar(<18 x float> inreg %a, i3
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -8588,10 +8823,18 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -8610,7 +8853,7 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8630,8 +8873,6 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v9i64_to_v18f32_scalar:
; VI: ; %bb.0:
@@ -8642,10 +8883,18 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -8664,7 +8913,7 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -8684,8 +8933,6 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v9i64_to_v18f32_scalar:
; GFX9: ; %bb.0:
@@ -8696,10 +8943,18 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -8718,7 +8973,7 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -8738,8 +8993,6 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v9i64_to_v18f32_scalar:
; GFX11: ; %bb.0:
@@ -8747,11 +9000,16 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
; GFX11-NEXT: s_add_u32 s26, s26, 3
@@ -8770,7 +9028,7 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8782,8 +9040,6 @@ define inreg <18 x float> @bitcast_v9i64_to_v18f32_scalar(<9 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8972,10 +9228,18 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
; SI-NEXT: v_add_f32_e64 v16, s52, 1.0
; SI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -8994,10 +9258,8 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9030,7 +9292,7 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -9083,10 +9345,18 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
; VI-NEXT: v_add_f32_e64 v16, s52, 1.0
; VI-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -9105,10 +9375,8 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -9141,7 +9409,7 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -9194,10 +9462,18 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX9-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -9216,10 +9492,8 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -9252,7 +9526,7 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -9306,11 +9580,16 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
; GFX11-NEXT: v_add_f32_e64 v16, s52, 1.0
; GFX11-NEXT: v_add_f32_e64 v15, s51, 1.0
@@ -9330,8 +9609,6 @@ define inreg <9 x double> @bitcast_v18f32_to_v9f64_scalar(<18 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -9526,10 +9803,18 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -9539,10 +9824,8 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9575,7 +9858,7 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -9628,10 +9911,18 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -9641,10 +9932,8 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -9677,7 +9966,7 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -9730,10 +10019,18 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -9743,10 +10040,8 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -9779,7 +10074,7 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -9833,11 +10128,16 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX11-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
@@ -9848,8 +10148,6 @@ define inreg <18 x float> @bitcast_v9f64_to_v18f32_scalar(<9 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -10422,7 +10720,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s75, s7, 16
@@ -10442,8 +10740,34 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v17, s5, 1.0
; SI-NEXT: v_add_f32_e64 v16, s4, 1.0
; SI-NEXT: v_add_f32_e64 v15, s7, 1.0
@@ -10480,28 +10804,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -10538,7 +10842,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s44
; SI-NEXT: v_mov_b32_e32 v25, s46
; SI-NEXT: v_mov_b32_e32 v26, s56
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
@@ -10604,7 +10908,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -10624,8 +10928,34 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v17, s6, 1.0
; VI-NEXT: v_add_f32_e64 v16, s7, 1.0
; VI-NEXT: v_add_f32_e64 v15, s8, 1.0
@@ -10662,28 +10992,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10720,7 +11030,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s12
; VI-NEXT: v_mov_b32_e32 v19, s11
; VI-NEXT: v_mov_b32_e32 v18, s10
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; VI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
@@ -10768,7 +11078,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -10788,8 +11098,34 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v17, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v15, s8, 1.0
@@ -10826,28 +11162,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10884,7 +11200,7 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -10927,12 +11243,12 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s29, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s28, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s26, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s25, 16
@@ -10949,9 +11265,34 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, s29, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v16, s28, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v15, s27, 1.0
@@ -10988,28 +11329,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -11027,8 +11348,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s6 :: v_dual_mov_b32 v18, s5
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s5 :: v_dual_mov_b32 v18, s4
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v34.l
@@ -11054,12 +11375,12 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s29, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s28, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s26, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s25, 16
@@ -11076,9 +11397,34 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v14, s28, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v15, s27, 1.0
@@ -11115,28 +11461,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v5, s17
@@ -11154,8 +11480,8 @@ define inreg <36 x i16> @bitcast_v18f32_to_v36i16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s6 :: v_dual_mov_b32 v18, s5
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s5 :: v_dual_mov_b32 v18, s4
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v36, 0xffff, v0
@@ -11999,7 +12325,7 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s15, s72, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -12055,8 +12381,17 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -12147,7 +12482,7 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -12187,9 +12522,6 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v36i16_to_v18f32_scalar:
; VI: ; %bb.0:
@@ -12237,7 +12569,7 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s15, s72, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -12293,8 +12625,17 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -12385,7 +12726,7 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -12425,9 +12766,6 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v36i16_to_v18f32_scalar:
; GFX9: ; %bb.0:
@@ -12487,10 +12825,18 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -12509,10 +12855,8 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -12545,7 +12889,7 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -12604,11 +12948,16 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -12628,8 +12977,6 @@ define inreg <18 x float> @bitcast_v36i16_to_v18f32_scalar(<36 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v16, s16, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v17, s17, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13187,7 +13534,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s75, s7, 16
@@ -13207,8 +13554,34 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v17, s5, 1.0
; SI-NEXT: v_add_f32_e64 v16, s4, 1.0
; SI-NEXT: v_add_f32_e64 v15, s7, 1.0
@@ -13245,28 +13618,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13303,7 +13656,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s44
; SI-NEXT: v_mov_b32_e32 v25, s46
; SI-NEXT: v_mov_b32_e32 v26, s56
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
@@ -13369,7 +13722,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -13389,8 +13742,34 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v17, s6, 1.0
; VI-NEXT: v_add_f32_e64 v16, s7, 1.0
; VI-NEXT: v_add_f32_e64 v15, s8, 1.0
@@ -13427,28 +13806,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13485,7 +13844,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s12
; VI-NEXT: v_mov_b32_e32 v19, s11
; VI-NEXT: v_mov_b32_e32 v18, s10
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; VI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
@@ -13533,7 +13892,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -13553,8 +13912,34 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v17, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v16, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v15, s8, 1.0
@@ -13591,28 +13976,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13649,7 +14014,7 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s12
; GFX9-NEXT: v_mov_b32_e32 v19, s11
; GFX9-NEXT: v_mov_b32_e32 v18, s10
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -13692,12 +14057,12 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s29, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s28, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s26, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s25, 16
@@ -13714,9 +14079,34 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, s29, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v16, s28, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v15, s27, 1.0
@@ -13753,28 +14143,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -13792,8 +14162,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s6 :: v_dual_mov_b32 v18, s5
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s5 :: v_dual_mov_b32 v18, s4
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v34.l
@@ -13819,12 +14189,12 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s29, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s28, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s26, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s25, 16
@@ -13841,9 +14211,34 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v13, s29, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v14, s28, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v15, s27, 1.0
@@ -13880,28 +14275,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v5, s17
@@ -13919,8 +14294,8 @@ define inreg <36 x half> @bitcast_v18f32_to_v36f16_scalar(<18 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s6 :: v_dual_mov_b32 v18, s5
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s5 :: v_dual_mov_b32 v18, s4
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v36, 0xffff, v0
@@ -14838,7 +15213,7 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s15, s13, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -14894,8 +15269,17 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s91
; SI-NEXT: v_cvt_f32_f16_e32 v2, s90
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
@@ -15040,11 +15424,8 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v18, v17
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -15077,7 +15458,7 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: .LBB35_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -15146,7 +15527,7 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s73, s15, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -15202,8 +15583,17 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v17, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s91
; VI-NEXT: v_mov_b32_e32 v2, s90
@@ -15277,11 +15667,8 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v18, v18, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v17, s6, v17
; VI-NEXT: v_or_b32_e32 v17, v17, v18
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -15314,7 +15701,7 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
+; VI-NEXT: .LBB35_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -15395,10 +15782,18 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v17, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v17 op_sel_hi:[1,0]
@@ -15418,10 +15813,8 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v15, s51, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v17 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -15454,7 +15847,7 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -15513,11 +15906,16 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -15537,8 +15935,6 @@ define inreg <18 x float> @bitcast_v36f16_to_v18f32_scalar(<36 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s16 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s17 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15731,10 +16127,18 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s8, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s9, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -15753,7 +16157,7 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15773,8 +16177,6 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v16, s7
; SI-NEXT: v_mov_b32_e32 v17, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v9i64_to_v9f64_scalar:
; VI: ; %bb.0:
@@ -15785,10 +16187,18 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -15807,7 +16217,7 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15827,8 +16237,6 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v9i64_to_v9f64_scalar:
; GFX9: ; %bb.0:
@@ -15839,10 +16247,18 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -15861,7 +16277,7 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -15881,8 +16297,6 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v9i64_to_v9f64_scalar:
; GFX11: ; %bb.0:
@@ -15890,11 +16304,16 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -15913,7 +16332,7 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_addc_u32 s27, s27, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -15924,8 +16343,6 @@ define inreg <9 x double> @bitcast_v9i64_to_v9f64_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16087,10 +16504,18 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -16100,10 +16525,8 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; SI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -16136,7 +16559,7 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s53, v32, 9
; SI-NEXT: v_readlane_b32 s52, v32, 8
; SI-NEXT: v_readlane_b32 s51, v32, 7
@@ -16189,10 +16612,18 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -16202,10 +16633,8 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; VI-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -16238,7 +16667,7 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s53, v32, 9
; VI-NEXT: v_readlane_b32 s52, v32, 8
; VI-NEXT: v_readlane_b32 s51, v32, 7
@@ -16291,10 +16720,18 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -16304,10 +16741,8 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; GFX9-NEXT: v_add_f64 v[12:13], s[48:49], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -16340,7 +16775,7 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -16394,11 +16829,16 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -16409,8 +16849,6 @@ define inreg <9 x i64> @bitcast_v9f64_to_v9i64_scalar(<9 x double> inreg %a, i32
; GFX11-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -17011,7 +17449,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s60, s5, 16
; SI-NEXT: s_lshr_b32 s61, s7, 16
@@ -17031,8 +17469,34 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -17069,7 +17533,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s9, s56, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff
; SI-NEXT: s_or_b32 s9, s11, s9
@@ -17143,26 +17607,6 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v16, s4
; SI-NEXT: v_mov_b32_e32 v17, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v9i64_to_v36i16_scalar:
; VI: ; %bb.0:
@@ -17173,7 +17617,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -17193,8 +17637,34 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -17231,7 +17701,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -17305,26 +17775,6 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v9i64_to_v36i16_scalar:
; GFX9: ; %bb.0:
@@ -17335,7 +17785,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -17355,8 +17805,34 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -17393,7 +17869,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s58
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s57
@@ -17431,26 +17907,6 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v9i64_to_v36i16_scalar:
; GFX11: ; %bb.0:
@@ -17458,7 +17914,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_mov_b32 s46, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-NEXT: s_lshr_b32 s5, s28, 16
@@ -17478,9 +17934,34 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
; GFX11-NEXT: s_add_u32 s26, s26, 3
@@ -17517,7 +17998,7 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s45
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s44
@@ -17547,26 +18028,6 @@ define inreg <36 x i16> @bitcast_v9i64_to_v36i16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, s6
; GFX11-NEXT: v_dual_mov_b32 v16, s5 :: v_dual_mov_b32 v17, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -18373,7 +18834,7 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s15, s72, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -18429,8 +18890,17 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -18521,7 +18991,7 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -18561,9 +19031,6 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v36i16_to_v9i64_scalar:
; VI: ; %bb.0:
@@ -18611,7 +19078,7 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; VI-NEXT: s_lshr_b32 s15, s72, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -18667,8 +19134,17 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -18759,7 +19235,7 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -18799,9 +19275,6 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v36i16_to_v9i64_scalar:
; GFX9: ; %bb.0:
@@ -18861,10 +19334,18 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -18883,10 +19364,8 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -18919,7 +19398,7 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -18978,11 +19457,16 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -19002,8 +19486,6 @@ define inreg <9 x i64> @bitcast_v36i16_to_v9i64_scalar(<36 x i16> inreg %a, i32
; GFX11-NEXT: v_pk_add_u16 v16, s16, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v17, s17, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -19589,7 +20071,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s60, s5, 16
; SI-NEXT: s_lshr_b32 s61, s7, 16
@@ -19609,8 +20091,34 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -19647,7 +20155,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s9, s56, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff
; SI-NEXT: s_or_b32 s9, s11, s9
@@ -19721,26 +20229,6 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v16, s4
; SI-NEXT: v_mov_b32_e32 v17, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v9i64_to_v36f16_scalar:
; VI: ; %bb.0:
@@ -19751,7 +20239,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s9, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_lshr_b32 s11, s7, 16
@@ -19771,8 +20259,34 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -19809,7 +20323,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: s_lshr_b32 s57, s18, 16
; VI-NEXT: s_lshr_b32 s58, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -19883,26 +20397,6 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v16, s7
; VI-NEXT: v_mov_b32_e32 v17, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v9i64_to_v36f16_scalar:
; GFX9: ; %bb.0:
@@ -19913,7 +20407,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s9, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s6, 16
; GFX9-NEXT: s_lshr_b32 s11, s7, 16
@@ -19933,8 +20427,34 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -19971,7 +20491,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s57, s18, 16
; GFX9-NEXT: s_lshr_b32 s58, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s58
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s57
@@ -20009,26 +20529,6 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v16, s7
; GFX9-NEXT: v_mov_b32_e32 v17, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v9i64_to_v36f16_scalar:
; GFX11: ; %bb.0:
@@ -20036,7 +20536,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: s_mov_b32 s46, 0
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-NEXT: s_lshr_b32 s5, s28, 16
@@ -20056,9 +20556,34 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s46, -1
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s28, s28, 3
; GFX11-NEXT: s_addc_u32 s29, s29, 0
; GFX11-NEXT: s_add_u32 s26, s26, 3
@@ -20095,7 +20620,7 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s43, s2, 16
; GFX11-NEXT: s_lshr_b32 s44, s1, 16
; GFX11-NEXT: s_lshr_b32 s45, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s45
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s44
@@ -20125,26 +20650,6 @@ define inreg <36 x half> @bitcast_v9i64_to_v36f16_scalar(<9 x i64> inreg %a, i32
; GFX11-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, s6
; GFX11-NEXT: v_dual_mov_b32 v16, s5 :: v_dual_mov_b32 v17, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -21025,7 +21530,7 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s15, s13, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -21081,8 +21586,17 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s91
; SI-NEXT: v_cvt_f32_f16_e32 v2, s90
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
@@ -21227,11 +21741,8 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v18, v17
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -21264,7 +21775,7 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -21333,7 +21844,7 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; VI-NEXT: s_lshr_b32 s73, s15, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -21389,8 +21900,17 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v17, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s91
; VI-NEXT: v_mov_b32_e32 v2, s90
@@ -21464,11 +21984,8 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; VI-NEXT: v_add_f16_sdwa v18, v18, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v17, s6, v17
; VI-NEXT: v_or_b32_e32 v17, v17, v18
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -21501,7 +22018,7 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
+; VI-NEXT: .LBB47_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -21582,10 +22099,18 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v17, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v17 op_sel_hi:[1,0]
@@ -21605,10 +22130,8 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; GFX9-NEXT: v_pk_add_f16 v15, s51, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v17 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -21641,7 +22164,7 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -21700,11 +22223,16 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -21724,8 +22252,6 @@ define inreg <9 x i64> @bitcast_v36f16_to_v9i64_scalar(<36 x half> inreg %a, i32
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s16 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s17 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22256,7 +22782,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s7, 16
; SI-NEXT: s_lshr_b32 s75, s5, 16
@@ -22276,8 +22802,34 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -22305,28 +22857,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v17, s7
@@ -22363,7 +22895,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v20, s12
; SI-NEXT: v_mov_b32_e32 v19, s10
; SI-NEXT: v_mov_b32_e32 v18, s8
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
@@ -22429,7 +22961,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s7, 16
; VI-NEXT: s_lshr_b32 s43, s6, 16
@@ -22449,8 +22981,34 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; VI-NEXT: s_lshr_b32 s58, s18, 16
; VI-NEXT: s_lshr_b32 s42, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s8, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s8, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -22478,28 +23036,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -22536,7 +23074,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v24, s12
; VI-NEXT: v_mov_b32_e32 v22, s11
; VI-NEXT: v_mov_b32_e32 v20, s10
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
@@ -22584,7 +23122,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s7, 16
; GFX9-NEXT: s_lshr_b32 s43, s6, 16
@@ -22604,8 +23142,34 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s58, s18, 16
; GFX9-NEXT: s_lshr_b32 s42, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s8, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -22633,28 +23197,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -22691,7 +23235,7 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v24, s12
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -22734,13 +23278,13 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s28, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s27, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s26, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s25, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s24, 16
@@ -22756,9 +23300,34 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
@@ -22786,28 +23355,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -22824,9 +23373,9 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s42 :: v_dual_mov_b32 v28, s9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s41 :: v_dual_mov_b32 v26, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s40 :: v_dual_mov_b32 v24, s7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s15 :: v_dual_mov_b32 v22, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s13 :: v_dual_mov_b32 v20, s5
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s15 :: v_dual_mov_b32 v22, s5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s13 :: v_dual_mov_b32 v20, s4
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v35.l
@@ -22852,13 +23401,13 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s28, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s27, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s26, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s25, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s24, 16
@@ -22874,9 +23423,34 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[13:14], s[28:29], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[26:27], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[24:25], 1.0
@@ -22904,28 +23478,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v19
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v19, s2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v10, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s18 :: v_dual_mov_b32 v8, s22
@@ -22943,8 +23497,8 @@ define inreg <36 x i16> @bitcast_v9f64_to_v36i16_scalar(<9 x double> inreg %a, i
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s15 :: v_dual_mov_b32 v31, s10
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s11 :: v_dual_mov_b32 v29, s9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s8 :: v_dual_mov_b32 v25, s7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s6 :: v_dual_mov_b32 v23, s5
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v23, s4
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
@@ -23788,7 +24342,7 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s15, s72, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -23844,8 +24398,17 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -23936,7 +24499,7 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -23976,9 +24539,6 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v36i16_to_v9f64_scalar:
; VI: ; %bb.0:
@@ -24026,7 +24586,7 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s15, s72, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -24082,8 +24642,17 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -24174,7 +24743,7 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -24214,9 +24783,6 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v36i16_to_v9f64_scalar:
; GFX9: ; %bb.0:
@@ -24276,10 +24842,18 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -24298,10 +24872,8 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_pk_add_u16 v15, s51, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v16, s52, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -24334,7 +24906,7 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -24393,11 +24965,16 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -24417,8 +24994,6 @@ define inreg <9 x double> @bitcast_v36i16_to_v9f64_scalar(<36 x i16> inreg %a, i
; GFX11-NEXT: v_pk_add_u16 v16, s16, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v17, s17, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -24949,7 +25524,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s7, 16
; SI-NEXT: s_lshr_b32 s75, s5, 16
@@ -24969,8 +25544,34 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[44:45], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[58:59], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[58:59], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr61
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr63
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[58:59], s[58:59], exec
+; SI-NEXT: s_cselect_b32 s9, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -24998,28 +25599,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr61
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr63
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v17, s7
@@ -25056,7 +25637,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, s12
; SI-NEXT: v_mov_b32_e32 v19, s10
; SI-NEXT: v_mov_b32_e32 v18, s8
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v25
@@ -25122,7 +25703,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s10, s7, 16
; VI-NEXT: s_lshr_b32 s43, s6, 16
@@ -25142,8 +25723,34 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s58, s18, 16
; VI-NEXT: s_lshr_b32 s42, s17, 16
; VI-NEXT: s_lshr_b32 s59, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s8, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s8, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -25171,28 +25778,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -25229,7 +25816,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s12
; VI-NEXT: v_mov_b32_e32 v22, s11
; VI-NEXT: v_mov_b32_e32 v20, s10
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
@@ -25277,7 +25864,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s10, s7, 16
; GFX9-NEXT: s_lshr_b32 s43, s6, 16
@@ -25297,8 +25884,34 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s58, s18, 16
; GFX9-NEXT: s_lshr_b32 s42, s17, 16
; GFX9-NEXT: s_lshr_b32 s59, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s8, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
@@ -25326,28 +25939,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -25384,7 +25977,7 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s12
; GFX9-NEXT: v_mov_b32_e32 v22, s11
; GFX9-NEXT: v_mov_b32_e32 v20, s10
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -25427,13 +26020,13 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s28, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s27, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s26, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s25, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s24, 16
@@ -25449,9 +26042,34 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[12:13], s[24:25], 1.0
@@ -25479,28 +26097,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -25517,9 +26115,9 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s42 :: v_dual_mov_b32 v28, s9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s41 :: v_dual_mov_b32 v26, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s40 :: v_dual_mov_b32 v24, s7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s15 :: v_dual_mov_b32 v22, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s13 :: v_dual_mov_b32 v20, s5
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s15 :: v_dual_mov_b32 v22, s5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s13 :: v_dual_mov_b32 v20, s4
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v35.l
@@ -25545,13 +26143,13 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s28, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s27, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s26, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s25, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s24, 16
@@ -25567,9 +26165,34 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[13:14], s[28:29], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[26:27], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[24:25], 1.0
@@ -25597,28 +26220,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v19
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v19, s2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v10, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s18 :: v_dual_mov_b32 v8, s22
@@ -25636,8 +26239,8 @@ define inreg <36 x half> @bitcast_v9f64_to_v36f16_scalar(<9 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v22, s15 :: v_dual_mov_b32 v31, s10
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s11 :: v_dual_mov_b32 v29, s9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s8 :: v_dual_mov_b32 v25, s7
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s6 :: v_dual_mov_b32 v23, s5
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v23, s4
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
@@ -26555,7 +27158,7 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s15, s13, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s91, 16
@@ -26611,8 +27214,17 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s91
; SI-NEXT: v_cvt_f32_f16_e32 v2, s90
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
@@ -26757,11 +27369,8 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v18, v17
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -26794,7 +27403,7 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -26863,7 +27472,7 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s73, s15, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -26919,8 +27528,17 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v17, 0x200
; VI-NEXT: v_mov_b32_e32 v0, s91
; VI-NEXT: v_mov_b32_e32 v2, s90
@@ -26994,11 +27612,8 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v18, v18, v17 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v17, s6, v17
; VI-NEXT: v_or_b32_e32 v17, v17, v18
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -27031,7 +27646,7 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
+; VI-NEXT: .LBB55_6: ; %end
; VI-NEXT: v_readlane_b32 s67, v32, 15
; VI-NEXT: v_readlane_b32 s66, v32, 14
; VI-NEXT: v_readlane_b32 s65, v32, 13
@@ -27112,10 +27727,18 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s51, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s52, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v17, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v17 op_sel_hi:[1,0]
@@ -27135,10 +27758,8 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v15, s51, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v16, s52, v17 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v17, s53, v17 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -27171,7 +27792,7 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
; GFX9-NEXT: v_readlane_b32 s52, v32, 8
; GFX9-NEXT: v_readlane_b32 s51, v32, 7
@@ -27230,11 +27851,16 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s15, s27, s15
; GFX11-NEXT: s_pack_ll_b32_b16 s16, s28, s42
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -27254,8 +27880,6 @@ define inreg <9 x double> @bitcast_v36f16_to_v9f64_scalar(<36 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v16, 0x200, s16 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v17, 0x200, s17 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -28065,91 +28689,89 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v18, s51, 9
; SI-NEXT: v_writelane_b32 v18, s52, 10
; SI-NEXT: v_writelane_b32 v18, s53, 11
-; SI-NEXT: v_writelane_b32 v18, s54, 12
-; SI-NEXT: v_writelane_b32 v18, s55, 13
-; SI-NEXT: v_writelane_b32 v18, s30, 14
-; SI-NEXT: v_writelane_b32 v18, s31, 15
-; SI-NEXT: v_readfirstlane_b32 s53, v3
-; SI-NEXT: v_readfirstlane_b32 s55, v2
-; SI-NEXT: v_readfirstlane_b32 s50, v1
-; SI-NEXT: v_readfirstlane_b32 s52, v0
-; SI-NEXT: s_lshr_b32 s30, s29, 16
-; SI-NEXT: s_lshr_b32 s49, s28, 16
+; SI-NEXT: v_writelane_b32 v18, s30, 12
+; SI-NEXT: v_writelane_b32 v18, s31, 13
+; SI-NEXT: v_readfirstlane_b32 s51, v3
+; SI-NEXT: v_readfirstlane_b32 s53, v2
+; SI-NEXT: v_readfirstlane_b32 s48, v1
+; SI-NEXT: v_readfirstlane_b32 s50, v0
+; SI-NEXT: s_lshr_b32 vcc_lo, s29, 16
+; SI-NEXT: s_lshr_b32 s39, s28, 16
; SI-NEXT: s_lshr_b32 s95, s27, 16
-; SI-NEXT: s_lshr_b32 s48, s26, 16
+; SI-NEXT: s_lshr_b32 s38, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s39, s24, 16
+; SI-NEXT: s_lshr_b32 s37, s24, 16
; SI-NEXT: s_lshr_b32 s93, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s22, 16
+; SI-NEXT: s_lshr_b32 s36, s22, 16
; SI-NEXT: s_lshr_b32 s92, s21, 16
-; SI-NEXT: s_lshr_b32 s37, s20, 16
+; SI-NEXT: s_lshr_b32 s35, s20, 16
; SI-NEXT: s_lshr_b32 s91, s19, 16
-; SI-NEXT: s_lshr_b32 s36, s18, 16
+; SI-NEXT: s_lshr_b32 s34, s18, 16
; SI-NEXT: s_lshr_b32 s90, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
-; SI-NEXT: s_lshr_b32 s34, s53, 16
-; SI-NEXT: s_lshr_b32 s54, s55, 16
-; SI-NEXT: s_lshr_b32 s31, s50, 16
-; SI-NEXT: s_lshr_b32 s51, s52, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
+; SI-NEXT: s_lshr_b32 s30, s51, 16
+; SI-NEXT: s_lshr_b32 s52, s53, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s48, 16
+; SI-NEXT: s_lshr_b32 s49, s50, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s90, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s35, 16
+; SI-NEXT: s_lshl_b32 s46, s31, 16
; SI-NEXT: s_or_b32 s47, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
; SI-NEXT: s_lshl_b32 s7, s91, 16
; SI-NEXT: s_or_b32 s40, s4, s46
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s36, 16
+; SI-NEXT: s_lshl_b32 s56, s34, 16
; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
; SI-NEXT: s_lshl_b32 s7, s92, 16
; SI-NEXT: s_or_b32 s14, s4, s56
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s58, s37, 16
+; SI-NEXT: s_lshl_b32 s58, s35, 16
; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s93, 16
; SI-NEXT: s_or_b32 s12, s4, s58
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s38, 16
+; SI-NEXT: s_lshl_b32 s60, s36, 16
; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
; SI-NEXT: s_lshl_b32 s7, s94, 16
; SI-NEXT: s_or_b32 s10, s4, s60
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s39, 16
+; SI-NEXT: s_lshl_b32 s62, s37, 16
; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
; SI-NEXT: s_lshl_b32 s7, s95, 16
; SI-NEXT: s_or_b32 s8, s4, s62
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s72, s48, 16
+; SI-NEXT: s_lshl_b32 s72, s38, 16
; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s30, 16
+; SI-NEXT: s_lshl_b32 s7, vcc_lo, 16
; SI-NEXT: s_or_b32 s6, s4, s72
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s49, 16
+; SI-NEXT: s_lshl_b32 s44, s39, 16
; SI-NEXT: s_or_b32 s45, s5, s7
-; SI-NEXT: s_and_b32 s5, s50, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s31, 16
+; SI-NEXT: s_and_b32 s5, s48, 0xffff
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s44
-; SI-NEXT: s_lshl_b32 s42, s51, 16
+; SI-NEXT: s_lshl_b32 s42, s49, 16
; SI-NEXT: s_or_b32 s43, s5, s7
-; SI-NEXT: s_and_b32 s5, s53, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
+; SI-NEXT: s_and_b32 s5, s51, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s30, 16
; SI-NEXT: s_lshr_b64 s[74:75], s[44:45], 16
-; SI-NEXT: s_and_b32 s44, s52, 0xffff
+; SI-NEXT: s_and_b32 s44, s50, 0xffff
; SI-NEXT: s_or_b32 s79, s5, s7
-; SI-NEXT: s_lshl_b32 s78, s54, 16
+; SI-NEXT: s_lshl_b32 s78, s52, 16
; SI-NEXT: s_or_b32 s44, s44, s42
; SI-NEXT: s_lshr_b64 s[76:77], s[42:43], 16
-; SI-NEXT: s_and_b32 s42, s55, 0xffff
+; SI-NEXT: s_and_b32 s42, s53, 0xffff
; SI-NEXT: s_mov_b32 s41, s47
; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
; SI-NEXT: s_mov_b32 s15, s57
@@ -28167,39 +28789,65 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s42, s42, s78
; SI-NEXT: s_mov_b32 s43, s79
; SI-NEXT: s_lshr_b64 s[78:79], s[78:79], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s55, s55, 3
-; SI-NEXT: s_and_b32 s4, s55, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s54, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_mov_b64 s[88:89], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[88:89], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[88:89], s[88:89], exec
+; SI-NEXT: s_cselect_b32 s47, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s47, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s53, s53, 3
-; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s53, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s52, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s52, s52, 3
-; SI-NEXT: s_add_i32 s43, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s52, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s51, 16
+; SI-NEXT: s_add_i32 s51, s51, 3
+; SI-NEXT: s_add_i32 s42, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s51, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s50, s50, 3
-; SI-NEXT: s_add_i32 s44, s4, 0x30000
+; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s50, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, s49, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s48, s48, 3
+; SI-NEXT: s_add_i32 s44, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s48, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s49, 16
+; SI-NEXT: s_lshl_b32 s5, s39, 16
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s30, 16
+; SI-NEXT: s_lshl_b32 s6, vcc_lo, 16
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s48, 16
+; SI-NEXT: s_lshl_b32 s7, s38, 16
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s27, 0xffff
@@ -28207,7 +28855,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s39, 16
+; SI-NEXT: s_lshl_b32 s9, s37, 16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s25, 0xffff
@@ -28215,7 +28863,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s38, 16
+; SI-NEXT: s_lshl_b32 s11, s36, 16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s23, 0xffff
@@ -28223,7 +28871,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_and_b32 s12, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s37, 16
+; SI-NEXT: s_lshl_b32 s13, s35, 16
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_and_b32 s13, s21, 0xffff
@@ -28231,7 +28879,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: s_and_b32 s14, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s36, 16
+; SI-NEXT: s_lshl_b32 s15, s34, 16
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s15, s19, 0xffff
@@ -28239,7 +28887,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s15, s18, s15
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s35, 16
+; SI-NEXT: s_lshl_b32 s18, s31, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s40, s16, 0x30000
@@ -28274,10 +28922,10 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s11, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s7, 16
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s45, 16
-; SI-NEXT: s_lshr_b32 s34, s43, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s45, 16
+; SI-NEXT: s_lshr_b32 s30, s43, 16
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s16, s40, 0xffff
; SI-NEXT: s_lshl_b32 s17, s46, 16
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -28318,21 +28966,21 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s18, s74, 16
; SI-NEXT: s_or_b32 s4, s4, s18
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s30, 16
+; SI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; SI-NEXT: s_or_b32 s5, s5, s18
; SI-NEXT: s_and_b32 s18, s44, 0xffff
; SI-NEXT: s_lshl_b32 s19, s76, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s31, 16
+; SI-NEXT: s_lshl_b32 s20, vcc_hi, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s42, 0xffff
; SI-NEXT: s_lshl_b32 s21, s78, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s34, 16
+; SI-NEXT: s_lshl_b32 s22, s30, 16
; SI-NEXT: s_or_b32 s21, s21, s22
-; SI-NEXT: v_readlane_b32 s30, v18, 14
+; SI-NEXT: v_readlane_b32 s30, v18, 12
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s14
@@ -28351,9 +28999,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v15, s19
; SI-NEXT: v_mov_b32_e32 v16, s20
; SI-NEXT: v_mov_b32_e32 v17, s21
-; SI-NEXT: v_readlane_b32 s31, v18, 15
-; SI-NEXT: v_readlane_b32 s55, v18, 13
-; SI-NEXT: v_readlane_b32 s54, v18, 12
+; SI-NEXT: v_readlane_b32 s31, v18, 13
; SI-NEXT: v_readlane_b32 s53, v18, 11
; SI-NEXT: v_readlane_b32 s52, v18, 10
; SI-NEXT: v_readlane_b32 s51, v18, 9
@@ -28371,26 +29017,6 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v36i16_to_v36f16_scalar:
; VI: ; %bb.0:
@@ -28419,10 +29045,18 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s42, s41, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s59, s59, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -28459,7 +29093,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s12, s12, 3
; VI-NEXT: s_add_i32 s8, s8, 3
; VI-NEXT: s_add_i32 s7, s7, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -28533,8 +29167,6 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v16, s11
; VI-NEXT: v_mov_b32_e32 v17, s7
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v36i16_to_v36f16_scalar:
; GFX9: ; %bb.0:
@@ -28563,10 +29195,18 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s56, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v4
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s59, s47
; GFX9-NEXT: v_pk_add_u16 v17, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s58, s46
@@ -28621,10 +29261,8 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v15
; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v17, s59
; GFX9-NEXT: v_mov_b32_e32 v16, s58
; GFX9-NEXT: v_mov_b32_e32 v15, s57
@@ -28661,7 +29299,7 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v33, s8
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -28724,11 +29362,16 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s46, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s46, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s28, s28, s44
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s27, s27, s43
@@ -28784,8 +29427,6 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v17
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v16, s28
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v14, s26
@@ -28852,11 +29493,16 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s46, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s46, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s28, s28, s44
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s27, s27, s43
@@ -28912,8 +29558,6 @@ define inreg <36 x half> @bitcast_v36i16_to_v36f16_scalar(<36 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v14
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v13
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s29 :: v_dual_mov_b32 v14, s28
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v16, s26
@@ -29622,10 +30266,18 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s44, s7, 16
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s59
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
@@ -29780,10 +30432,8 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[22:23], v[12:13], 16
; SI-NEXT: v_lshr_b64 v[20:21], v[14:15], 16
; SI-NEXT: v_lshr_b64 v[18:19], v[16:17], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v37, s47
; SI-NEXT: v_mov_b32_e32 v51, s41
; SI-NEXT: v_mov_b32_e32 v38, s40
@@ -29824,7 +30474,7 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v22, s43
; SI-NEXT: v_mov_b32_e32 v20, s44
; SI-NEXT: v_mov_b32_e32 v18, s42
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v34
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v36
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -29915,10 +30565,18 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s59, s58, 16
; VI-NEXT: v_readfirstlane_b32 s4, v4
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v18, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v18
; VI-NEXT: v_add_f16_e32 v35, s43, v18
@@ -29956,10 +30614,8 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v19, s47, v18
; VI-NEXT: v_add_f16_e32 v17, s44, v18
; VI-NEXT: v_add_f16_e32 v18, s45, v18
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v18, s45
; VI-NEXT: v_mov_b32_e32 v17, s44
; VI-NEXT: v_mov_b32_e32 v19, s47
@@ -29996,7 +30652,7 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v35, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; VI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
@@ -30062,10 +30718,18 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s56, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v4
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s59, s47
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v17, s4, v0 op_sel_hi:[1,0]
@@ -30121,10 +30785,8 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v15
; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v17
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v17, s59
; GFX9-NEXT: v_mov_b32_e32 v16, s58
; GFX9-NEXT: v_mov_b32_e32 v15, s57
@@ -30161,7 +30823,7 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v33, s8
; GFX9-NEXT: v_mov_b32_e32 v34, s7
; GFX9-NEXT: v_mov_b32_e32 v35, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -30224,11 +30886,16 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s46, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s46, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s46, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s28, s28, s44
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s27, s27, s43
@@ -30284,8 +30951,6 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v17
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v16, s28
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v14, s26
@@ -30352,11 +31017,16 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s46, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s46, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s46, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s46, s46, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s46, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s46, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s28, s28, s44
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s27, s27, s43
@@ -30412,8 +31082,6 @@ define inreg <36 x i16> @bitcast_v36f16_to_v36i16_scalar(<36 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v14
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v18, 16, v13
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, s29 :: v_dual_mov_b32 v14, s28
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s27 :: v_dual_mov_b32 v16, s26
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
index 69562f9d0a70d..39f3e88052c88 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll
@@ -168,10 +168,18 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -192,7 +200,7 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -214,8 +222,6 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v20i32_to_v20f32_scalar:
; VI: ; %bb.0:
@@ -228,10 +234,18 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -252,7 +266,7 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -274,8 +288,6 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v20i32_to_v20f32_scalar:
; GFX9: ; %bb.0:
@@ -288,10 +300,18 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -312,7 +332,7 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -334,8 +354,6 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v20i32_to_v20f32_scalar:
; GFX11: ; %bb.0:
@@ -345,11 +363,16 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
@@ -370,7 +393,7 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -383,8 +406,6 @@ define inreg <20 x float> @bitcast_v20i32_to_v20f32_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -584,10 +605,18 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
; SI-NEXT: v_add_f32_e64 v18, s54, 1.0
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -608,10 +637,8 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -644,7 +671,7 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -703,10 +730,18 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
; VI-NEXT: v_add_f32_e64 v18, s54, 1.0
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -727,10 +762,8 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -763,7 +796,7 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -822,10 +855,18 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -846,10 +887,8 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -882,7 +921,7 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -942,11 +981,16 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX11-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -968,8 +1012,6 @@ define inreg <20 x i32> @bitcast_v20f32_to_v20i32_scalar(<20 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1184,10 +1226,18 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1208,7 +1258,7 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1230,8 +1280,6 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v20i32_to_v10i64_scalar:
; VI: ; %bb.0:
@@ -1244,10 +1292,18 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1268,7 +1324,7 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1290,8 +1346,6 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v20i32_to_v10i64_scalar:
; GFX9: ; %bb.0:
@@ -1304,10 +1358,18 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1328,7 +1390,7 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1350,8 +1412,6 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v20i32_to_v10i64_scalar:
; GFX11: ; %bb.0:
@@ -1361,11 +1421,16 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
@@ -1386,7 +1451,7 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1399,8 +1464,6 @@ define inreg <10 x i64> @bitcast_v20i32_to_v10i64_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1585,10 +1648,18 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1609,7 +1680,7 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1631,8 +1702,6 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v10i64_to_v20i32_scalar:
; VI: ; %bb.0:
@@ -1645,10 +1714,18 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -1669,7 +1746,7 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1691,8 +1768,6 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v10i64_to_v20i32_scalar:
; GFX9: ; %bb.0:
@@ -1705,10 +1780,18 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -1729,7 +1812,7 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1751,8 +1834,6 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v10i64_to_v20i32_scalar:
; GFX11: ; %bb.0:
@@ -1762,11 +1843,16 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
@@ -1787,7 +1873,7 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1800,8 +1886,6 @@ define inreg <20 x i32> @bitcast_v10i64_to_v20i32_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1981,10 +2065,18 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2005,7 +2097,7 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2027,8 +2119,6 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v20i32_to_v10f64_scalar:
; VI: ; %bb.0:
@@ -2041,10 +2131,18 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2065,7 +2163,7 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2087,8 +2185,6 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v20i32_to_v10f64_scalar:
; GFX9: ; %bb.0:
@@ -2101,10 +2197,18 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2125,7 +2229,7 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2147,8 +2251,6 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v20i32_to_v10f64_scalar:
; GFX11: ; %bb.0:
@@ -2158,11 +2260,16 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
@@ -2183,7 +2290,7 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2196,8 +2303,6 @@ define inreg <10 x double> @bitcast_v20i32_to_v10f64_scalar(<20 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2367,10 +2472,18 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -2381,10 +2494,8 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2417,7 +2528,7 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2476,10 +2587,18 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -2490,10 +2609,8 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -2526,7 +2643,7 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2585,10 +2702,18 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -2599,10 +2724,8 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -2635,7 +2758,7 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -2695,11 +2818,16 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -2711,8 +2839,6 @@ define inreg <20 x i32> @bitcast_v10f64_to_v20i32_scalar(<10 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -3355,7 +3481,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s5, 16
; SI-NEXT: s_lshr_b32 s73, s7, 16
@@ -3377,8 +3503,36 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3419,7 +3573,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s11, s60, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff
; SI-NEXT: s_or_b32 s11, s13, s11
@@ -3501,28 +3655,6 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v18, s4
; SI-NEXT: v_mov_b32_e32 v19, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v20i32_to_v40i16_scalar:
; VI: ; %bb.0:
@@ -3535,7 +3667,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -3557,8 +3689,36 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -3599,7 +3759,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -3681,28 +3841,6 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v20i32_to_v40i16_scalar:
; GFX9: ; %bb.0:
@@ -3715,7 +3853,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -3737,8 +3875,36 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -3779,7 +3945,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s61
@@ -3821,28 +3987,6 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v20i32_to_v40i16_scalar:
; GFX11: ; %bb.0:
@@ -3852,7 +3996,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_mov_b32 s58, 0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s6, s4, 16
; GFX11-NEXT: s_lshr_b32 s7, s5, 16
@@ -3874,9 +4018,36 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
@@ -3917,7 +4088,7 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s57
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s56
@@ -3950,28 +4121,6 @@ define inreg <40 x i16> @bitcast_v20i32_to_v40i16_scalar(<20 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v16, s9 :: v_dual_mov_b32 v17, s8
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4889,7 +5038,7 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s76, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -4951,8 +5100,17 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -5053,7 +5211,7 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s55, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -5095,9 +5253,6 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v40i16_to_v20i32_scalar:
; VI: ; %bb.0:
@@ -5105,26 +5260,24 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v20, s34, 0
-; VI-NEXT: v_writelane_b32 v20, s35, 1
-; VI-NEXT: v_writelane_b32 v20, s36, 2
-; VI-NEXT: v_writelane_b32 v20, s37, 3
-; VI-NEXT: v_writelane_b32 v20, s38, 4
-; VI-NEXT: v_writelane_b32 v20, s39, 5
-; VI-NEXT: v_writelane_b32 v20, s48, 6
-; VI-NEXT: v_writelane_b32 v20, s49, 7
-; VI-NEXT: v_writelane_b32 v20, s50, 8
-; VI-NEXT: v_writelane_b32 v20, s51, 9
-; VI-NEXT: v_writelane_b32 v20, s52, 10
-; VI-NEXT: v_writelane_b32 v20, s53, 11
-; VI-NEXT: v_writelane_b32 v20, s54, 12
-; VI-NEXT: v_writelane_b32 v20, s55, 13
-; VI-NEXT: v_writelane_b32 v20, s64, 14
-; VI-NEXT: v_writelane_b32 v20, s65, 15
-; VI-NEXT: v_writelane_b32 v20, s66, 16
-; VI-NEXT: v_writelane_b32 v20, s67, 17
-; VI-NEXT: v_writelane_b32 v20, s30, 18
-; VI-NEXT: v_writelane_b32 v20, s31, 19
+; VI-NEXT: v_writelane_b32 v20, s36, 0
+; VI-NEXT: v_writelane_b32 v20, s37, 1
+; VI-NEXT: v_writelane_b32 v20, s38, 2
+; VI-NEXT: v_writelane_b32 v20, s39, 3
+; VI-NEXT: v_writelane_b32 v20, s48, 4
+; VI-NEXT: v_writelane_b32 v20, s49, 5
+; VI-NEXT: v_writelane_b32 v20, s50, 6
+; VI-NEXT: v_writelane_b32 v20, s51, 7
+; VI-NEXT: v_writelane_b32 v20, s52, 8
+; VI-NEXT: v_writelane_b32 v20, s53, 9
+; VI-NEXT: v_writelane_b32 v20, s54, 10
+; VI-NEXT: v_writelane_b32 v20, s55, 11
+; VI-NEXT: v_writelane_b32 v20, s64, 12
+; VI-NEXT: v_writelane_b32 v20, s65, 13
+; VI-NEXT: v_writelane_b32 v20, s66, 14
+; VI-NEXT: v_writelane_b32 v20, s67, 15
+; VI-NEXT: v_writelane_b32 v20, s30, 16
+; VI-NEXT: v_writelane_b32 v20, s31, 17
; VI-NEXT: v_readfirstlane_b32 s7, v5
; VI-NEXT: v_readfirstlane_b32 s9, v4
; VI-NEXT: v_readfirstlane_b32 s12, v3
@@ -5141,10 +5294,10 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s12, 16
@@ -5153,19 +5306,19 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -5215,26 +5368,35 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -5317,8 +5479,8 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s55, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v20, 18
+; VI-NEXT: .LBB15_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v20, 16
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -5339,33 +5501,28 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v17, s53
; VI-NEXT: v_mov_b32_e32 v18, s54
; VI-NEXT: v_mov_b32_e32 v19, s55
-; VI-NEXT: v_readlane_b32 s31, v20, 19
-; VI-NEXT: v_readlane_b32 s67, v20, 17
-; VI-NEXT: v_readlane_b32 s66, v20, 16
-; VI-NEXT: v_readlane_b32 s65, v20, 15
-; VI-NEXT: v_readlane_b32 s64, v20, 14
-; VI-NEXT: v_readlane_b32 s55, v20, 13
-; VI-NEXT: v_readlane_b32 s54, v20, 12
-; VI-NEXT: v_readlane_b32 s53, v20, 11
-; VI-NEXT: v_readlane_b32 s52, v20, 10
-; VI-NEXT: v_readlane_b32 s51, v20, 9
-; VI-NEXT: v_readlane_b32 s50, v20, 8
-; VI-NEXT: v_readlane_b32 s49, v20, 7
-; VI-NEXT: v_readlane_b32 s48, v20, 6
-; VI-NEXT: v_readlane_b32 s39, v20, 5
-; VI-NEXT: v_readlane_b32 s38, v20, 4
-; VI-NEXT: v_readlane_b32 s37, v20, 3
-; VI-NEXT: v_readlane_b32 s36, v20, 2
-; VI-NEXT: v_readlane_b32 s35, v20, 1
-; VI-NEXT: v_readlane_b32 s34, v20, 0
+; VI-NEXT: v_readlane_b32 s31, v20, 17
+; VI-NEXT: v_readlane_b32 s67, v20, 15
+; VI-NEXT: v_readlane_b32 s66, v20, 14
+; VI-NEXT: v_readlane_b32 s65, v20, 13
+; VI-NEXT: v_readlane_b32 s64, v20, 12
+; VI-NEXT: v_readlane_b32 s55, v20, 11
+; VI-NEXT: v_readlane_b32 s54, v20, 10
+; VI-NEXT: v_readlane_b32 s53, v20, 9
+; VI-NEXT: v_readlane_b32 s52, v20, 8
+; VI-NEXT: v_readlane_b32 s51, v20, 7
+; VI-NEXT: v_readlane_b32 s50, v20, 6
+; VI-NEXT: v_readlane_b32 s49, v20, 5
+; VI-NEXT: v_readlane_b32 s48, v20, 4
+; VI-NEXT: v_readlane_b32 s39, v20, 3
+; VI-NEXT: v_readlane_b32 s38, v20, 2
+; VI-NEXT: v_readlane_b32 s37, v20, 1
+; VI-NEXT: v_readlane_b32 s36, v20, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v40i16_to_v20i32_scalar:
; GFX9: ; %bb.0:
@@ -5433,10 +5590,18 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -5457,10 +5622,8 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -5493,7 +5656,7 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -5560,11 +5723,16 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -5586,8 +5754,6 @@ define inreg <20 x i32> @bitcast_v40i16_to_v20i32_scalar(<40 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v18, s18, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v19, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6213,7 +6379,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s5, 16
; SI-NEXT: s_lshr_b32 s73, s7, 16
@@ -6235,8 +6401,36 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -6277,7 +6471,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s11, s60, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff
; SI-NEXT: s_or_b32 s11, s13, s11
@@ -6359,28 +6553,6 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v18, s4
; SI-NEXT: v_mov_b32_e32 v19, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v20i32_to_v40f16_scalar:
; VI: ; %bb.0:
@@ -6393,7 +6565,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -6415,8 +6587,36 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -6457,7 +6657,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -6539,28 +6739,6 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v20i32_to_v40f16_scalar:
; GFX9: ; %bb.0:
@@ -6573,7 +6751,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -6595,12 +6773,40 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
-; GFX9-NEXT: s_add_i32 s6, s6, 3
-; GFX9-NEXT: s_add_i32 s7, s7, 3
-; GFX9-NEXT: s_add_i32 s8, s8, 3
-; GFX9-NEXT: s_add_i32 s9, s9, 3
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_add_i32 s6, s6, 3
+; GFX9-NEXT: s_add_i32 s7, s7, 3
+; GFX9-NEXT: s_add_i32 s8, s8, 3
+; GFX9-NEXT: s_add_i32 s9, s9, 3
; GFX9-NEXT: s_add_i32 s10, s10, 3
; GFX9-NEXT: s_add_i32 s11, s11, 3
; GFX9-NEXT: s_add_i32 s29, s29, 3
@@ -6637,7 +6843,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s61
@@ -6679,28 +6885,6 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v20i32_to_v40f16_scalar:
; GFX11: ; %bb.0:
@@ -6710,7 +6894,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_mov_b32 s58, 0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s6, s4, 16
; GFX11-NEXT: s_lshr_b32 s7, s5, 16
@@ -6732,9 +6916,36 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s29, s29, 3
@@ -6775,7 +6986,7 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s57
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s56
@@ -6808,28 +7019,6 @@ define inreg <40 x half> @bitcast_v20i32_to_v40f16_scalar(<20 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v16, s9 :: v_dual_mov_b32 v17, s8
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7832,7 +8021,7 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s74, s73, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -7894,8 +8083,17 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s95
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
@@ -8056,11 +8254,8 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v18, v19, v18
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v20
; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -8093,7 +8288,7 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
+; SI-NEXT: .LBB19_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -8122,26 +8317,24 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v32, s34, 0
-; VI-NEXT: v_writelane_b32 v32, s35, 1
-; VI-NEXT: v_writelane_b32 v32, s36, 2
-; VI-NEXT: v_writelane_b32 v32, s37, 3
-; VI-NEXT: v_writelane_b32 v32, s38, 4
-; VI-NEXT: v_writelane_b32 v32, s39, 5
-; VI-NEXT: v_writelane_b32 v32, s48, 6
-; VI-NEXT: v_writelane_b32 v32, s49, 7
-; VI-NEXT: v_writelane_b32 v32, s50, 8
-; VI-NEXT: v_writelane_b32 v32, s51, 9
-; VI-NEXT: v_writelane_b32 v32, s52, 10
-; VI-NEXT: v_writelane_b32 v32, s53, 11
-; VI-NEXT: v_writelane_b32 v32, s54, 12
-; VI-NEXT: v_writelane_b32 v32, s55, 13
-; VI-NEXT: v_writelane_b32 v32, s64, 14
-; VI-NEXT: v_writelane_b32 v32, s65, 15
-; VI-NEXT: v_writelane_b32 v32, s66, 16
-; VI-NEXT: v_writelane_b32 v32, s67, 17
-; VI-NEXT: v_writelane_b32 v32, s30, 18
-; VI-NEXT: v_writelane_b32 v32, s31, 19
+; VI-NEXT: v_writelane_b32 v32, s36, 0
+; VI-NEXT: v_writelane_b32 v32, s37, 1
+; VI-NEXT: v_writelane_b32 v32, s38, 2
+; VI-NEXT: v_writelane_b32 v32, s39, 3
+; VI-NEXT: v_writelane_b32 v32, s48, 4
+; VI-NEXT: v_writelane_b32 v32, s49, 5
+; VI-NEXT: v_writelane_b32 v32, s50, 6
+; VI-NEXT: v_writelane_b32 v32, s51, 7
+; VI-NEXT: v_writelane_b32 v32, s52, 8
+; VI-NEXT: v_writelane_b32 v32, s53, 9
+; VI-NEXT: v_writelane_b32 v32, s54, 10
+; VI-NEXT: v_writelane_b32 v32, s55, 11
+; VI-NEXT: v_writelane_b32 v32, s64, 12
+; VI-NEXT: v_writelane_b32 v32, s65, 13
+; VI-NEXT: v_writelane_b32 v32, s66, 14
+; VI-NEXT: v_writelane_b32 v32, s67, 15
+; VI-NEXT: v_writelane_b32 v32, s30, 16
+; VI-NEXT: v_writelane_b32 v32, s31, 17
; VI-NEXT: v_readfirstlane_b32 s6, v5
; VI-NEXT: v_readfirstlane_b32 s8, v4
; VI-NEXT: v_readfirstlane_b32 s11, v3
@@ -8158,10 +8351,10 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s12, s11, 16
@@ -8170,19 +8363,19 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -8232,22 +8425,31 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v19, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s35
-; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v0, s31
+; VI-NEXT: v_mov_b32_e32 v2, s30
; VI-NEXT: v_add_f16_sdwa v0, v0, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v19
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v19
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s31
+; VI-NEXT: v_mov_b32_e32 v2, vcc_hi
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v19
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s30
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: v_add_f16_sdwa v3, v3, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v19
; VI-NEXT: v_or_b32_e32 v3, v4, v3
@@ -8315,11 +8517,8 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v20, v20, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s6, v19
; VI-NEXT: v_or_b32_e32 v19, v19, v20
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -8352,27 +8551,25 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 18
-; VI-NEXT: v_readlane_b32 s31, v32, 19
-; VI-NEXT: v_readlane_b32 s67, v32, 17
-; VI-NEXT: v_readlane_b32 s66, v32, 16
-; VI-NEXT: v_readlane_b32 s65, v32, 15
-; VI-NEXT: v_readlane_b32 s64, v32, 14
-; VI-NEXT: v_readlane_b32 s55, v32, 13
-; VI-NEXT: v_readlane_b32 s54, v32, 12
-; VI-NEXT: v_readlane_b32 s53, v32, 11
-; VI-NEXT: v_readlane_b32 s52, v32, 10
-; VI-NEXT: v_readlane_b32 s51, v32, 9
-; VI-NEXT: v_readlane_b32 s50, v32, 8
-; VI-NEXT: v_readlane_b32 s49, v32, 7
-; VI-NEXT: v_readlane_b32 s48, v32, 6
-; VI-NEXT: v_readlane_b32 s39, v32, 5
-; VI-NEXT: v_readlane_b32 s38, v32, 4
-; VI-NEXT: v_readlane_b32 s37, v32, 3
-; VI-NEXT: v_readlane_b32 s36, v32, 2
-; VI-NEXT: v_readlane_b32 s35, v32, 1
-; VI-NEXT: v_readlane_b32 s34, v32, 0
+; VI-NEXT: .LBB19_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 16
+; VI-NEXT: v_readlane_b32 s31, v32, 17
+; VI-NEXT: v_readlane_b32 s67, v32, 15
+; VI-NEXT: v_readlane_b32 s66, v32, 14
+; VI-NEXT: v_readlane_b32 s65, v32, 13
+; VI-NEXT: v_readlane_b32 s64, v32, 12
+; VI-NEXT: v_readlane_b32 s55, v32, 11
+; VI-NEXT: v_readlane_b32 s54, v32, 10
+; VI-NEXT: v_readlane_b32 s53, v32, 9
+; VI-NEXT: v_readlane_b32 s52, v32, 8
+; VI-NEXT: v_readlane_b32 s51, v32, 7
+; VI-NEXT: v_readlane_b32 s50, v32, 6
+; VI-NEXT: v_readlane_b32 s49, v32, 5
+; VI-NEXT: v_readlane_b32 s48, v32, 4
+; VI-NEXT: v_readlane_b32 s39, v32, 3
+; VI-NEXT: v_readlane_b32 s38, v32, 2
+; VI-NEXT: v_readlane_b32 s37, v32, 1
+; VI-NEXT: v_readlane_b32 s36, v32, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -8445,10 +8642,18 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v19, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v19 op_sel_hi:[1,0]
@@ -8470,10 +8675,8 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v17, s53, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s54, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v19, s55, v19 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -8506,7 +8709,7 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -8573,11 +8776,16 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -8599,8 +8807,6 @@ define inreg <20 x i32> @bitcast_v40f16_to_v20i32_scalar(<40 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s18 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8818,10 +9024,18 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
; SI-NEXT: v_add_f32_e64 v18, s54, 1.0
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -8842,10 +9056,8 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -8878,7 +9090,7 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -8937,10 +9149,18 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
; VI-NEXT: v_add_f32_e64 v18, s54, 1.0
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -8961,10 +9181,8 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -8997,7 +9215,7 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -9056,10 +9274,18 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -9080,10 +9306,8 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -9116,7 +9340,7 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -9176,11 +9400,16 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX11-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -9202,8 +9431,6 @@ define inreg <10 x i64> @bitcast_v20f32_to_v10i64_scalar(<20 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -9423,10 +9650,18 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -9447,7 +9682,7 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -9469,8 +9704,6 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v10i64_to_v20f32_scalar:
; VI: ; %bb.0:
@@ -9483,10 +9716,18 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -9507,7 +9748,7 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -9529,8 +9770,6 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v10i64_to_v20f32_scalar:
; GFX9: ; %bb.0:
@@ -9543,10 +9782,18 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -9567,7 +9814,7 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -9589,8 +9836,6 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v10i64_to_v20f32_scalar:
; GFX11: ; %bb.0:
@@ -9600,11 +9845,16 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
@@ -9625,7 +9875,7 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9638,8 +9888,6 @@ define inreg <20 x float> @bitcast_v10i64_to_v20f32_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9839,10 +10087,18 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
; SI-NEXT: v_add_f32_e64 v18, s54, 1.0
; SI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -9863,10 +10119,8 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9899,7 +10153,7 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -9958,10 +10212,18 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
; VI-NEXT: v_add_f32_e64 v18, s54, 1.0
; VI-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -9982,10 +10244,8 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -10018,7 +10278,7 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10077,10 +10337,18 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -10101,10 +10369,8 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10137,7 +10403,7 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -10197,11 +10463,16 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
; GFX11-NEXT: v_add_f32_e64 v18, s54, 1.0
; GFX11-NEXT: v_add_f32_e64 v17, s53, 1.0
@@ -10223,8 +10494,6 @@ define inreg <10 x double> @bitcast_v20f32_to_v10f64_scalar(<20 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -10429,10 +10698,18 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -10443,10 +10720,8 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10479,7 +10754,7 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10538,10 +10813,18 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -10552,10 +10835,8 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -10588,7 +10869,7 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10647,10 +10928,18 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -10661,10 +10950,8 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10697,7 +10984,7 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -10757,11 +11044,16 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
@@ -10773,8 +11065,6 @@ define inreg <20 x float> @bitcast_v10f64_to_v20f32_scalar(<10 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -11397,7 +11687,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s89, s5, 16
; SI-NEXT: s_lshr_b32 s88, s7, 16
@@ -11419,8 +11709,36 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v19, s5, 1.0
; SI-NEXT: v_add_f32_e64 v18, s4, 1.0
; SI-NEXT: v_add_f32_e64 v17, s7, 1.0
@@ -11461,30 +11779,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -11525,7 +11821,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v27, s56
; SI-NEXT: v_mov_b32_e32 v28, s58
; SI-NEXT: v_mov_b32_e32 v29, s60
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
@@ -11599,7 +11895,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -11621,8 +11917,36 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s6, 1.0
; VI-NEXT: v_add_f32_e64 v18, s7, 1.0
; VI-NEXT: v_add_f32_e64 v17, s8, 1.0
@@ -11663,30 +11987,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v38, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11727,7 +12029,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s14
; VI-NEXT: v_mov_b32_e32 v21, s13
; VI-NEXT: v_mov_b32_e32 v20, s12
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; VI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -11781,7 +12083,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -11803,8 +12105,36 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s8, 1.0
@@ -11845,30 +12175,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11909,7 +12217,7 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -11958,12 +12266,12 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s27, 16
@@ -11982,9 +12290,36 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s56, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v18, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, s29, 1.0
@@ -12025,30 +12360,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -12068,8 +12381,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s7 :: v_dual_mov_b32 v20, s6
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v39.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v38.l
@@ -12099,12 +12412,12 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s27, 16
@@ -12123,9 +12436,36 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s56, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v15, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v16, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v17, s29, 1.0
@@ -12166,30 +12506,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v3, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v9, s17
@@ -12209,8 +12527,8 @@ define inreg <40 x i16> @bitcast_v20f32_to_v40i16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s7 :: v_dual_mov_b32 v20, s6
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v1
@@ -13169,7 +13487,7 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s76, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -13231,8 +13549,17 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -13333,7 +13660,7 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s55, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -13375,9 +13702,6 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v40i16_to_v20f32_scalar:
; VI: ; %bb.0:
@@ -13385,26 +13709,24 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v20, s34, 0
-; VI-NEXT: v_writelane_b32 v20, s35, 1
-; VI-NEXT: v_writelane_b32 v20, s36, 2
-; VI-NEXT: v_writelane_b32 v20, s37, 3
-; VI-NEXT: v_writelane_b32 v20, s38, 4
-; VI-NEXT: v_writelane_b32 v20, s39, 5
-; VI-NEXT: v_writelane_b32 v20, s48, 6
-; VI-NEXT: v_writelane_b32 v20, s49, 7
-; VI-NEXT: v_writelane_b32 v20, s50, 8
-; VI-NEXT: v_writelane_b32 v20, s51, 9
-; VI-NEXT: v_writelane_b32 v20, s52, 10
-; VI-NEXT: v_writelane_b32 v20, s53, 11
-; VI-NEXT: v_writelane_b32 v20, s54, 12
-; VI-NEXT: v_writelane_b32 v20, s55, 13
-; VI-NEXT: v_writelane_b32 v20, s64, 14
-; VI-NEXT: v_writelane_b32 v20, s65, 15
-; VI-NEXT: v_writelane_b32 v20, s66, 16
-; VI-NEXT: v_writelane_b32 v20, s67, 17
-; VI-NEXT: v_writelane_b32 v20, s30, 18
-; VI-NEXT: v_writelane_b32 v20, s31, 19
+; VI-NEXT: v_writelane_b32 v20, s36, 0
+; VI-NEXT: v_writelane_b32 v20, s37, 1
+; VI-NEXT: v_writelane_b32 v20, s38, 2
+; VI-NEXT: v_writelane_b32 v20, s39, 3
+; VI-NEXT: v_writelane_b32 v20, s48, 4
+; VI-NEXT: v_writelane_b32 v20, s49, 5
+; VI-NEXT: v_writelane_b32 v20, s50, 6
+; VI-NEXT: v_writelane_b32 v20, s51, 7
+; VI-NEXT: v_writelane_b32 v20, s52, 8
+; VI-NEXT: v_writelane_b32 v20, s53, 9
+; VI-NEXT: v_writelane_b32 v20, s54, 10
+; VI-NEXT: v_writelane_b32 v20, s55, 11
+; VI-NEXT: v_writelane_b32 v20, s64, 12
+; VI-NEXT: v_writelane_b32 v20, s65, 13
+; VI-NEXT: v_writelane_b32 v20, s66, 14
+; VI-NEXT: v_writelane_b32 v20, s67, 15
+; VI-NEXT: v_writelane_b32 v20, s30, 16
+; VI-NEXT: v_writelane_b32 v20, s31, 17
; VI-NEXT: v_readfirstlane_b32 s7, v5
; VI-NEXT: v_readfirstlane_b32 s9, v4
; VI-NEXT: v_readfirstlane_b32 s12, v3
@@ -13421,10 +13743,10 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s12, 16
@@ -13433,19 +13755,19 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -13495,26 +13817,35 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -13597,8 +13928,8 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s55, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v20, 18
+; VI-NEXT: .LBB31_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v20, 16
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -13619,33 +13950,28 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v17, s53
; VI-NEXT: v_mov_b32_e32 v18, s54
; VI-NEXT: v_mov_b32_e32 v19, s55
-; VI-NEXT: v_readlane_b32 s31, v20, 19
-; VI-NEXT: v_readlane_b32 s67, v20, 17
-; VI-NEXT: v_readlane_b32 s66, v20, 16
-; VI-NEXT: v_readlane_b32 s65, v20, 15
-; VI-NEXT: v_readlane_b32 s64, v20, 14
-; VI-NEXT: v_readlane_b32 s55, v20, 13
-; VI-NEXT: v_readlane_b32 s54, v20, 12
-; VI-NEXT: v_readlane_b32 s53, v20, 11
-; VI-NEXT: v_readlane_b32 s52, v20, 10
-; VI-NEXT: v_readlane_b32 s51, v20, 9
-; VI-NEXT: v_readlane_b32 s50, v20, 8
-; VI-NEXT: v_readlane_b32 s49, v20, 7
-; VI-NEXT: v_readlane_b32 s48, v20, 6
-; VI-NEXT: v_readlane_b32 s39, v20, 5
-; VI-NEXT: v_readlane_b32 s38, v20, 4
-; VI-NEXT: v_readlane_b32 s37, v20, 3
-; VI-NEXT: v_readlane_b32 s36, v20, 2
-; VI-NEXT: v_readlane_b32 s35, v20, 1
-; VI-NEXT: v_readlane_b32 s34, v20, 0
+; VI-NEXT: v_readlane_b32 s31, v20, 17
+; VI-NEXT: v_readlane_b32 s67, v20, 15
+; VI-NEXT: v_readlane_b32 s66, v20, 14
+; VI-NEXT: v_readlane_b32 s65, v20, 13
+; VI-NEXT: v_readlane_b32 s64, v20, 12
+; VI-NEXT: v_readlane_b32 s55, v20, 11
+; VI-NEXT: v_readlane_b32 s54, v20, 10
+; VI-NEXT: v_readlane_b32 s53, v20, 9
+; VI-NEXT: v_readlane_b32 s52, v20, 8
+; VI-NEXT: v_readlane_b32 s51, v20, 7
+; VI-NEXT: v_readlane_b32 s50, v20, 6
+; VI-NEXT: v_readlane_b32 s49, v20, 5
+; VI-NEXT: v_readlane_b32 s48, v20, 4
+; VI-NEXT: v_readlane_b32 s39, v20, 3
+; VI-NEXT: v_readlane_b32 s38, v20, 2
+; VI-NEXT: v_readlane_b32 s37, v20, 1
+; VI-NEXT: v_readlane_b32 s36, v20, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v40i16_to_v20f32_scalar:
; GFX9: ; %bb.0:
@@ -13713,10 +14039,18 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -13737,10 +14071,8 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -13773,7 +14105,7 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -13840,11 +14172,16 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -13866,8 +14203,6 @@ define inreg <20 x float> @bitcast_v40i16_to_v20f32_scalar(<40 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v18, s18, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v19, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -14473,7 +14808,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s89, s5, 16
; SI-NEXT: s_lshr_b32 s88, s7, 16
@@ -14495,8 +14830,36 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v19, s5, 1.0
; SI-NEXT: v_add_f32_e64 v18, s4, 1.0
; SI-NEXT: v_add_f32_e64 v17, s7, 1.0
@@ -14537,30 +14900,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -14601,7 +14942,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v27, s56
; SI-NEXT: v_mov_b32_e32 v28, s58
; SI-NEXT: v_mov_b32_e32 v29, s60
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
@@ -14675,7 +15016,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -14697,8 +15038,36 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v19, s6, 1.0
; VI-NEXT: v_add_f32_e64 v18, s7, 1.0
; VI-NEXT: v_add_f32_e64 v17, s8, 1.0
@@ -14739,30 +15108,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v38, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14803,7 +15150,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s14
; VI-NEXT: v_mov_b32_e32 v21, s13
; VI-NEXT: v_mov_b32_e32 v20, s12
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; VI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -14857,7 +15204,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -14879,8 +15226,36 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v19, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v18, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v17, s8, 1.0
@@ -14921,30 +15296,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14985,7 +15338,7 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s14
; GFX9-NEXT: v_mov_b32_e32 v21, s13
; GFX9-NEXT: v_mov_b32_e32 v20, s12
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -15034,12 +15387,12 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s27, 16
@@ -15058,9 +15411,36 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s56, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v18, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, s29, 1.0
@@ -15085,46 +15465,24 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -15144,8 +15502,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s7 :: v_dual_mov_b32 v20, s6
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v39.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v38.l
@@ -15175,12 +15533,12 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s27, 16
@@ -15199,9 +15557,36 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s56, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v15, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v16, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v17, s29, 1.0
@@ -15242,30 +15627,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v3, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v9, s17
@@ -15285,8 +15648,8 @@ define inreg <40 x half> @bitcast_v20f32_to_v40f16_scalar(<20 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s8 :: v_dual_mov_b32 v20, s7
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s7 :: v_dual_mov_b32 v20, s6
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v1
@@ -16330,7 +16693,7 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s74, s73, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -16392,8 +16755,17 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s95
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
@@ -16554,11 +16926,8 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v18, v19, v18
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v20
; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -16591,7 +16960,7 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
+; SI-NEXT: .LBB35_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -16620,26 +16989,24 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v32, s34, 0
-; VI-NEXT: v_writelane_b32 v32, s35, 1
-; VI-NEXT: v_writelane_b32 v32, s36, 2
-; VI-NEXT: v_writelane_b32 v32, s37, 3
-; VI-NEXT: v_writelane_b32 v32, s38, 4
-; VI-NEXT: v_writelane_b32 v32, s39, 5
-; VI-NEXT: v_writelane_b32 v32, s48, 6
-; VI-NEXT: v_writelane_b32 v32, s49, 7
-; VI-NEXT: v_writelane_b32 v32, s50, 8
-; VI-NEXT: v_writelane_b32 v32, s51, 9
-; VI-NEXT: v_writelane_b32 v32, s52, 10
-; VI-NEXT: v_writelane_b32 v32, s53, 11
-; VI-NEXT: v_writelane_b32 v32, s54, 12
-; VI-NEXT: v_writelane_b32 v32, s55, 13
-; VI-NEXT: v_writelane_b32 v32, s64, 14
-; VI-NEXT: v_writelane_b32 v32, s65, 15
-; VI-NEXT: v_writelane_b32 v32, s66, 16
-; VI-NEXT: v_writelane_b32 v32, s67, 17
-; VI-NEXT: v_writelane_b32 v32, s30, 18
-; VI-NEXT: v_writelane_b32 v32, s31, 19
+; VI-NEXT: v_writelane_b32 v32, s36, 0
+; VI-NEXT: v_writelane_b32 v32, s37, 1
+; VI-NEXT: v_writelane_b32 v32, s38, 2
+; VI-NEXT: v_writelane_b32 v32, s39, 3
+; VI-NEXT: v_writelane_b32 v32, s48, 4
+; VI-NEXT: v_writelane_b32 v32, s49, 5
+; VI-NEXT: v_writelane_b32 v32, s50, 6
+; VI-NEXT: v_writelane_b32 v32, s51, 7
+; VI-NEXT: v_writelane_b32 v32, s52, 8
+; VI-NEXT: v_writelane_b32 v32, s53, 9
+; VI-NEXT: v_writelane_b32 v32, s54, 10
+; VI-NEXT: v_writelane_b32 v32, s55, 11
+; VI-NEXT: v_writelane_b32 v32, s64, 12
+; VI-NEXT: v_writelane_b32 v32, s65, 13
+; VI-NEXT: v_writelane_b32 v32, s66, 14
+; VI-NEXT: v_writelane_b32 v32, s67, 15
+; VI-NEXT: v_writelane_b32 v32, s30, 16
+; VI-NEXT: v_writelane_b32 v32, s31, 17
; VI-NEXT: v_readfirstlane_b32 s6, v5
; VI-NEXT: v_readfirstlane_b32 s8, v4
; VI-NEXT: v_readfirstlane_b32 s11, v3
@@ -16656,10 +17023,10 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s12, s11, 16
@@ -16668,19 +17035,19 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -16730,22 +17097,31 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v19, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s35
-; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v0, s31
+; VI-NEXT: v_mov_b32_e32 v2, s30
; VI-NEXT: v_add_f16_sdwa v0, v0, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v19
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v19
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s31
+; VI-NEXT: v_mov_b32_e32 v2, vcc_hi
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v19
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s30
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: v_add_f16_sdwa v3, v3, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v19
; VI-NEXT: v_or_b32_e32 v3, v4, v3
@@ -16813,11 +17189,8 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v20, v20, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s6, v19
; VI-NEXT: v_or_b32_e32 v19, v19, v20
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -16850,27 +17223,25 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 18
-; VI-NEXT: v_readlane_b32 s31, v32, 19
-; VI-NEXT: v_readlane_b32 s67, v32, 17
-; VI-NEXT: v_readlane_b32 s66, v32, 16
-; VI-NEXT: v_readlane_b32 s65, v32, 15
-; VI-NEXT: v_readlane_b32 s64, v32, 14
-; VI-NEXT: v_readlane_b32 s55, v32, 13
-; VI-NEXT: v_readlane_b32 s54, v32, 12
-; VI-NEXT: v_readlane_b32 s53, v32, 11
-; VI-NEXT: v_readlane_b32 s52, v32, 10
-; VI-NEXT: v_readlane_b32 s51, v32, 9
-; VI-NEXT: v_readlane_b32 s50, v32, 8
-; VI-NEXT: v_readlane_b32 s49, v32, 7
-; VI-NEXT: v_readlane_b32 s48, v32, 6
-; VI-NEXT: v_readlane_b32 s39, v32, 5
-; VI-NEXT: v_readlane_b32 s38, v32, 4
-; VI-NEXT: v_readlane_b32 s37, v32, 3
-; VI-NEXT: v_readlane_b32 s36, v32, 2
-; VI-NEXT: v_readlane_b32 s35, v32, 1
-; VI-NEXT: v_readlane_b32 s34, v32, 0
+; VI-NEXT: .LBB35_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 16
+; VI-NEXT: v_readlane_b32 s31, v32, 17
+; VI-NEXT: v_readlane_b32 s67, v32, 15
+; VI-NEXT: v_readlane_b32 s66, v32, 14
+; VI-NEXT: v_readlane_b32 s65, v32, 13
+; VI-NEXT: v_readlane_b32 s64, v32, 12
+; VI-NEXT: v_readlane_b32 s55, v32, 11
+; VI-NEXT: v_readlane_b32 s54, v32, 10
+; VI-NEXT: v_readlane_b32 s53, v32, 9
+; VI-NEXT: v_readlane_b32 s52, v32, 8
+; VI-NEXT: v_readlane_b32 s51, v32, 7
+; VI-NEXT: v_readlane_b32 s50, v32, 6
+; VI-NEXT: v_readlane_b32 s49, v32, 5
+; VI-NEXT: v_readlane_b32 s48, v32, 4
+; VI-NEXT: v_readlane_b32 s39, v32, 3
+; VI-NEXT: v_readlane_b32 s38, v32, 2
+; VI-NEXT: v_readlane_b32 s37, v32, 1
+; VI-NEXT: v_readlane_b32 s36, v32, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -16943,10 +17314,18 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v19, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v19 op_sel_hi:[1,0]
@@ -16968,10 +17347,8 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v17, s53, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s54, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v19, s55, v19 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -17004,7 +17381,7 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -17071,11 +17448,16 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -17097,8 +17479,6 @@ define inreg <20 x float> @bitcast_v40f16_to_v20f32_scalar(<40 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s18 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -17301,10 +17681,18 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s10, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s11, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -17325,7 +17713,7 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -17347,8 +17735,6 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s7
; SI-NEXT: v_mov_b32_e32 v19, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v10i64_to_v10f64_scalar:
; VI: ; %bb.0:
@@ -17361,10 +17747,18 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -17385,7 +17779,7 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17407,8 +17801,6 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v10i64_to_v10f64_scalar:
; GFX9: ; %bb.0:
@@ -17421,10 +17813,18 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -17445,7 +17845,7 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17467,8 +17867,6 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v10i64_to_v10f64_scalar:
; GFX11: ; %bb.0:
@@ -17478,11 +17876,16 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_mov_b32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s6, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -17503,7 +17906,7 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s29, s29, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -17515,8 +17918,6 @@ define inreg <10 x double> @bitcast_v10i64_to_v10f64_scalar(<10 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v17, s29
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -17686,10 +18087,18 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -17700,10 +18109,8 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; SI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -17736,7 +18143,7 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -17795,10 +18202,18 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -17809,10 +18224,8 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; VI-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -17845,7 +18258,7 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -17904,10 +18317,18 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -17918,10 +18339,8 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[14:15], s[50:51], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -17954,7 +18373,7 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -18014,11 +18433,16 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -18030,8 +18454,6 @@ define inreg <10 x i64> @bitcast_v10f64_to_v10i64_scalar(<10 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -18684,7 +19106,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s5, 16
; SI-NEXT: s_lshr_b32 s73, s7, 16
@@ -18706,8 +19128,36 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -18748,7 +19198,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s11, s60, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff
; SI-NEXT: s_or_b32 s11, s13, s11
@@ -18830,28 +19280,6 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v18, s4
; SI-NEXT: v_mov_b32_e32 v19, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v10i64_to_v40i16_scalar:
; VI: ; %bb.0:
@@ -18864,7 +19292,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -18886,8 +19314,36 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -18928,7 +19384,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -19010,28 +19466,6 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v10i64_to_v40i16_scalar:
; GFX9: ; %bb.0:
@@ -19044,7 +19478,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -19066,8 +19500,36 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -19108,7 +19570,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s61
@@ -19150,28 +19612,6 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v10i64_to_v40i16_scalar:
; GFX11: ; %bb.0:
@@ -19181,7 +19621,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_mov_b32 s58, 0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s6, s4, 16
; GFX11-NEXT: s_lshr_b32 s7, s5, 16
@@ -19203,9 +19643,36 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
@@ -19246,7 +19713,7 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s57
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s56
@@ -19270,37 +19737,15 @@ define inreg <40 x i16> @bitcast_v10i64_to_v40i16_scalar(<10 x i64> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s4, s4, s6
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
-; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
-; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
-; GFX11-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v11, s14
-; GFX11-NEXT: v_dual_mov_b32 v12, s13 :: v_dual_mov_b32 v13, s12
-; GFX11-NEXT: v_dual_mov_b32 v14, s11 :: v_dual_mov_b32 v15, s10
-; GFX11-NEXT: v_dual_mov_b32 v16, s9 :: v_dual_mov_b32 v17, s8
-; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB41_2
+; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
+; GFX11-NEXT: v_dual_mov_b32 v6, s18 :: v_dual_mov_b32 v7, s19
+; GFX11-NEXT: v_dual_mov_b32 v8, s20 :: v_dual_mov_b32 v9, s21
+; GFX11-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v11, s14
+; GFX11-NEXT: v_dual_mov_b32 v12, s13 :: v_dual_mov_b32 v13, s12
+; GFX11-NEXT: v_dual_mov_b32 v14, s11 :: v_dual_mov_b32 v15, s10
+; GFX11-NEXT: v_dual_mov_b32 v16, s9 :: v_dual_mov_b32 v17, s8
+; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -20218,7 +20663,7 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s76, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -20280,8 +20725,17 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -20382,7 +20836,7 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s55, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
+; SI-NEXT: .LBB43_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -20424,9 +20878,6 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v40i16_to_v10i64_scalar:
; VI: ; %bb.0:
@@ -20434,26 +20885,24 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v20, s34, 0
-; VI-NEXT: v_writelane_b32 v20, s35, 1
-; VI-NEXT: v_writelane_b32 v20, s36, 2
-; VI-NEXT: v_writelane_b32 v20, s37, 3
-; VI-NEXT: v_writelane_b32 v20, s38, 4
-; VI-NEXT: v_writelane_b32 v20, s39, 5
-; VI-NEXT: v_writelane_b32 v20, s48, 6
-; VI-NEXT: v_writelane_b32 v20, s49, 7
-; VI-NEXT: v_writelane_b32 v20, s50, 8
-; VI-NEXT: v_writelane_b32 v20, s51, 9
-; VI-NEXT: v_writelane_b32 v20, s52, 10
-; VI-NEXT: v_writelane_b32 v20, s53, 11
-; VI-NEXT: v_writelane_b32 v20, s54, 12
-; VI-NEXT: v_writelane_b32 v20, s55, 13
-; VI-NEXT: v_writelane_b32 v20, s64, 14
-; VI-NEXT: v_writelane_b32 v20, s65, 15
-; VI-NEXT: v_writelane_b32 v20, s66, 16
-; VI-NEXT: v_writelane_b32 v20, s67, 17
-; VI-NEXT: v_writelane_b32 v20, s30, 18
-; VI-NEXT: v_writelane_b32 v20, s31, 19
+; VI-NEXT: v_writelane_b32 v20, s36, 0
+; VI-NEXT: v_writelane_b32 v20, s37, 1
+; VI-NEXT: v_writelane_b32 v20, s38, 2
+; VI-NEXT: v_writelane_b32 v20, s39, 3
+; VI-NEXT: v_writelane_b32 v20, s48, 4
+; VI-NEXT: v_writelane_b32 v20, s49, 5
+; VI-NEXT: v_writelane_b32 v20, s50, 6
+; VI-NEXT: v_writelane_b32 v20, s51, 7
+; VI-NEXT: v_writelane_b32 v20, s52, 8
+; VI-NEXT: v_writelane_b32 v20, s53, 9
+; VI-NEXT: v_writelane_b32 v20, s54, 10
+; VI-NEXT: v_writelane_b32 v20, s55, 11
+; VI-NEXT: v_writelane_b32 v20, s64, 12
+; VI-NEXT: v_writelane_b32 v20, s65, 13
+; VI-NEXT: v_writelane_b32 v20, s66, 14
+; VI-NEXT: v_writelane_b32 v20, s67, 15
+; VI-NEXT: v_writelane_b32 v20, s30, 16
+; VI-NEXT: v_writelane_b32 v20, s31, 17
; VI-NEXT: v_readfirstlane_b32 s7, v5
; VI-NEXT: v_readfirstlane_b32 s9, v4
; VI-NEXT: v_readfirstlane_b32 s12, v3
@@ -20470,10 +20919,10 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s12, 16
@@ -20482,19 +20931,19 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -20544,26 +20993,35 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -20646,8 +21104,8 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s55, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v20, 18
+; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v20, 16
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -20668,33 +21126,28 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v17, s53
; VI-NEXT: v_mov_b32_e32 v18, s54
; VI-NEXT: v_mov_b32_e32 v19, s55
-; VI-NEXT: v_readlane_b32 s31, v20, 19
-; VI-NEXT: v_readlane_b32 s67, v20, 17
-; VI-NEXT: v_readlane_b32 s66, v20, 16
-; VI-NEXT: v_readlane_b32 s65, v20, 15
-; VI-NEXT: v_readlane_b32 s64, v20, 14
-; VI-NEXT: v_readlane_b32 s55, v20, 13
-; VI-NEXT: v_readlane_b32 s54, v20, 12
-; VI-NEXT: v_readlane_b32 s53, v20, 11
-; VI-NEXT: v_readlane_b32 s52, v20, 10
-; VI-NEXT: v_readlane_b32 s51, v20, 9
-; VI-NEXT: v_readlane_b32 s50, v20, 8
-; VI-NEXT: v_readlane_b32 s49, v20, 7
-; VI-NEXT: v_readlane_b32 s48, v20, 6
-; VI-NEXT: v_readlane_b32 s39, v20, 5
-; VI-NEXT: v_readlane_b32 s38, v20, 4
-; VI-NEXT: v_readlane_b32 s37, v20, 3
-; VI-NEXT: v_readlane_b32 s36, v20, 2
-; VI-NEXT: v_readlane_b32 s35, v20, 1
-; VI-NEXT: v_readlane_b32 s34, v20, 0
+; VI-NEXT: v_readlane_b32 s31, v20, 17
+; VI-NEXT: v_readlane_b32 s67, v20, 15
+; VI-NEXT: v_readlane_b32 s66, v20, 14
+; VI-NEXT: v_readlane_b32 s65, v20, 13
+; VI-NEXT: v_readlane_b32 s64, v20, 12
+; VI-NEXT: v_readlane_b32 s55, v20, 11
+; VI-NEXT: v_readlane_b32 s54, v20, 10
+; VI-NEXT: v_readlane_b32 s53, v20, 9
+; VI-NEXT: v_readlane_b32 s52, v20, 8
+; VI-NEXT: v_readlane_b32 s51, v20, 7
+; VI-NEXT: v_readlane_b32 s50, v20, 6
+; VI-NEXT: v_readlane_b32 s49, v20, 5
+; VI-NEXT: v_readlane_b32 s48, v20, 4
+; VI-NEXT: v_readlane_b32 s39, v20, 3
+; VI-NEXT: v_readlane_b32 s38, v20, 2
+; VI-NEXT: v_readlane_b32 s37, v20, 1
+; VI-NEXT: v_readlane_b32 s36, v20, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v40i16_to_v10i64_scalar:
; GFX9: ; %bb.0:
@@ -20762,10 +21215,18 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -20786,10 +21247,8 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -20822,7 +21281,7 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -20889,11 +21348,16 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -20915,8 +21379,6 @@ define inreg <10 x i64> @bitcast_v40i16_to_v10i64_scalar(<40 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v18, s18, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v19, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -21552,7 +22014,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s9, v1
; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s72, s5, 16
; SI-NEXT: s_lshr_b32 s73, s7, 16
@@ -21574,8 +22036,36 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -21616,7 +22106,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s11, s60, 16
; SI-NEXT: s_and_b32 s13, s16, 0xffff
; SI-NEXT: s_or_b32 s11, s13, s11
@@ -21698,28 +22188,6 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v18, s4
; SI-NEXT: v_mov_b32_e32 v19, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v10i64_to_v40f16_scalar:
; VI: ; %bb.0:
@@ -21732,7 +22200,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s10, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s11, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s6, 16
; VI-NEXT: s_lshr_b32 s13, s7, 16
@@ -21754,8 +22222,36 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -21796,7 +22292,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s61, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -21878,28 +22374,6 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v18, s7
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v10i64_to_v40f16_scalar:
; GFX9: ; %bb.0:
@@ -21912,7 +22386,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s10, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s11, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s6, 16
; GFX9-NEXT: s_lshr_b32 s13, s7, 16
@@ -21934,8 +22408,36 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -21976,7 +22478,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s61, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s61
@@ -22018,28 +22520,6 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v18, s7
; GFX9-NEXT: v_mov_b32_e32 v19, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v10i64_to_v40f16_scalar:
; GFX11: ; %bb.0:
@@ -22049,7 +22529,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s5, v0
; GFX11-NEXT: s_mov_b32 s58, 0
; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s6, s4, 16
; GFX11-NEXT: s_lshr_b32 s7, s5, 16
@@ -22071,9 +22551,36 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s58, -1
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s28, s28, 3
@@ -22114,7 +22621,7 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s47, s2, 16
; GFX11-NEXT: s_lshr_b32 s56, s1, 16
; GFX11-NEXT: s_lshr_b32 s57, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s57
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s56
@@ -22147,28 +22654,6 @@ define inreg <40 x half> @bitcast_v10i64_to_v40f16_scalar(<10 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v16, s9 :: v_dual_mov_b32 v17, s8
; GFX11-NEXT: v_dual_mov_b32 v18, s5 :: v_dual_mov_b32 v19, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -23171,7 +23656,7 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s74, s73, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -23233,8 +23718,17 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s95
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
@@ -23395,11 +23889,8 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v18, v19, v18
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v20
; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -23432,7 +23923,7 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
+; SI-NEXT: .LBB47_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -23461,26 +23952,24 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v32, s34, 0
-; VI-NEXT: v_writelane_b32 v32, s35, 1
-; VI-NEXT: v_writelane_b32 v32, s36, 2
-; VI-NEXT: v_writelane_b32 v32, s37, 3
-; VI-NEXT: v_writelane_b32 v32, s38, 4
-; VI-NEXT: v_writelane_b32 v32, s39, 5
-; VI-NEXT: v_writelane_b32 v32, s48, 6
-; VI-NEXT: v_writelane_b32 v32, s49, 7
-; VI-NEXT: v_writelane_b32 v32, s50, 8
-; VI-NEXT: v_writelane_b32 v32, s51, 9
-; VI-NEXT: v_writelane_b32 v32, s52, 10
-; VI-NEXT: v_writelane_b32 v32, s53, 11
-; VI-NEXT: v_writelane_b32 v32, s54, 12
-; VI-NEXT: v_writelane_b32 v32, s55, 13
-; VI-NEXT: v_writelane_b32 v32, s64, 14
-; VI-NEXT: v_writelane_b32 v32, s65, 15
-; VI-NEXT: v_writelane_b32 v32, s66, 16
-; VI-NEXT: v_writelane_b32 v32, s67, 17
-; VI-NEXT: v_writelane_b32 v32, s30, 18
-; VI-NEXT: v_writelane_b32 v32, s31, 19
+; VI-NEXT: v_writelane_b32 v32, s36, 0
+; VI-NEXT: v_writelane_b32 v32, s37, 1
+; VI-NEXT: v_writelane_b32 v32, s38, 2
+; VI-NEXT: v_writelane_b32 v32, s39, 3
+; VI-NEXT: v_writelane_b32 v32, s48, 4
+; VI-NEXT: v_writelane_b32 v32, s49, 5
+; VI-NEXT: v_writelane_b32 v32, s50, 6
+; VI-NEXT: v_writelane_b32 v32, s51, 7
+; VI-NEXT: v_writelane_b32 v32, s52, 8
+; VI-NEXT: v_writelane_b32 v32, s53, 9
+; VI-NEXT: v_writelane_b32 v32, s54, 10
+; VI-NEXT: v_writelane_b32 v32, s55, 11
+; VI-NEXT: v_writelane_b32 v32, s64, 12
+; VI-NEXT: v_writelane_b32 v32, s65, 13
+; VI-NEXT: v_writelane_b32 v32, s66, 14
+; VI-NEXT: v_writelane_b32 v32, s67, 15
+; VI-NEXT: v_writelane_b32 v32, s30, 16
+; VI-NEXT: v_writelane_b32 v32, s31, 17
; VI-NEXT: v_readfirstlane_b32 s6, v5
; VI-NEXT: v_readfirstlane_b32 s8, v4
; VI-NEXT: v_readfirstlane_b32 s11, v3
@@ -23497,10 +23986,10 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s12, s11, 16
@@ -23509,19 +23998,19 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -23571,22 +24060,31 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v19, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s35
-; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v0, s31
+; VI-NEXT: v_mov_b32_e32 v2, s30
; VI-NEXT: v_add_f16_sdwa v0, v0, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v19
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v19
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s31
+; VI-NEXT: v_mov_b32_e32 v2, vcc_hi
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v19
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s30
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: v_add_f16_sdwa v3, v3, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v19
; VI-NEXT: v_or_b32_e32 v3, v4, v3
@@ -23654,11 +24152,8 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v20, v20, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s6, v19
; VI-NEXT: v_or_b32_e32 v19, v19, v20
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -23691,27 +24186,25 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 18
-; VI-NEXT: v_readlane_b32 s31, v32, 19
-; VI-NEXT: v_readlane_b32 s67, v32, 17
-; VI-NEXT: v_readlane_b32 s66, v32, 16
-; VI-NEXT: v_readlane_b32 s65, v32, 15
-; VI-NEXT: v_readlane_b32 s64, v32, 14
-; VI-NEXT: v_readlane_b32 s55, v32, 13
-; VI-NEXT: v_readlane_b32 s54, v32, 12
-; VI-NEXT: v_readlane_b32 s53, v32, 11
-; VI-NEXT: v_readlane_b32 s52, v32, 10
-; VI-NEXT: v_readlane_b32 s51, v32, 9
-; VI-NEXT: v_readlane_b32 s50, v32, 8
-; VI-NEXT: v_readlane_b32 s49, v32, 7
-; VI-NEXT: v_readlane_b32 s48, v32, 6
-; VI-NEXT: v_readlane_b32 s39, v32, 5
-; VI-NEXT: v_readlane_b32 s38, v32, 4
-; VI-NEXT: v_readlane_b32 s37, v32, 3
-; VI-NEXT: v_readlane_b32 s36, v32, 2
-; VI-NEXT: v_readlane_b32 s35, v32, 1
-; VI-NEXT: v_readlane_b32 s34, v32, 0
+; VI-NEXT: .LBB47_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 16
+; VI-NEXT: v_readlane_b32 s31, v32, 17
+; VI-NEXT: v_readlane_b32 s67, v32, 15
+; VI-NEXT: v_readlane_b32 s66, v32, 14
+; VI-NEXT: v_readlane_b32 s65, v32, 13
+; VI-NEXT: v_readlane_b32 s64, v32, 12
+; VI-NEXT: v_readlane_b32 s55, v32, 11
+; VI-NEXT: v_readlane_b32 s54, v32, 10
+; VI-NEXT: v_readlane_b32 s53, v32, 9
+; VI-NEXT: v_readlane_b32 s52, v32, 8
+; VI-NEXT: v_readlane_b32 s51, v32, 7
+; VI-NEXT: v_readlane_b32 s50, v32, 6
+; VI-NEXT: v_readlane_b32 s49, v32, 5
+; VI-NEXT: v_readlane_b32 s48, v32, 4
+; VI-NEXT: v_readlane_b32 s39, v32, 3
+; VI-NEXT: v_readlane_b32 s38, v32, 2
+; VI-NEXT: v_readlane_b32 s37, v32, 1
+; VI-NEXT: v_readlane_b32 s36, v32, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -23784,10 +24277,18 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v19, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v19 op_sel_hi:[1,0]
@@ -23809,10 +24310,8 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v17, s53, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s54, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v19, s55, v19 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -23845,7 +24344,7 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -23912,11 +24411,16 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -23938,8 +24442,6 @@ define inreg <10 x i64> @bitcast_v40f16_to_v10i64_scalar(<40 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s18 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -24515,7 +25017,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s89, s9, 16
; SI-NEXT: s_lshr_b32 s88, s7, 16
@@ -24537,8 +25039,36 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -24566,33 +25096,11 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v11
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -24633,7 +25141,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s14
; SI-NEXT: v_mov_b32_e32 v21, s12
; SI-NEXT: v_mov_b32_e32 v20, s10
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
@@ -24707,7 +25215,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s9, 16
; VI-NEXT: s_lshr_b32 s46, s8, 16
@@ -24729,8 +25237,36 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s62, s18, 16
; VI-NEXT: s_lshr_b32 s45, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[10:11], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[10:11], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; VI-NEXT: s_cselect_b32 s10, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s10, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -24761,30 +25297,8 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -24825,7 +25339,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s14
; VI-NEXT: v_mov_b32_e32 v24, s13
; VI-NEXT: v_mov_b32_e32 v22, s12
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
; VI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
@@ -24879,7 +25393,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s9, 16
; GFX9-NEXT: s_lshr_b32 s46, s8, 16
@@ -24901,8 +25415,36 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s62, s18, 16
; GFX9-NEXT: s_lshr_b32 s45, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[10:11], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s10, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -24933,30 +25475,8 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -24997,7 +25517,7 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s14
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -25046,13 +25566,13 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s26, 16
@@ -25070,9 +25590,36 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
@@ -25103,30 +25650,8 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -25145,9 +25670,9 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s45 :: v_dual_mov_b32 v30, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s44 :: v_dual_mov_b32 v28, s10
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s43 :: v_dual_mov_b32 v26, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s42 :: v_dual_mov_b32 v24, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s40 :: v_dual_mov_b32 v22, s7
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s42 :: v_dual_mov_b32 v24, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s40 :: v_dual_mov_b32 v22, s6
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v37.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v39.l
@@ -25177,13 +25702,13 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s26, 16
@@ -25201,9 +25726,36 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[28:29], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[26:27], 1.0
@@ -25234,30 +25786,8 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v21, s16
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s20 :: v_dual_mov_b32 v12, s22
@@ -25277,8 +25807,8 @@ define inreg <40 x i16> @bitcast_v10f64_to_v40i16_scalar(<10 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s15 :: v_dual_mov_b32 v33, s12
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s13 :: v_dual_mov_b32 v31, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s10 :: v_dual_mov_b32 v27, s9
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s8 :: v_dual_mov_b32 v25, s7
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v25, s6
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v21, 0xffff, v21
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v0
@@ -26237,7 +26767,7 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s76, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -26299,8 +26829,17 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -26401,7 +26940,7 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s55, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -26443,9 +26982,6 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v40i16_to_v10f64_scalar:
; VI: ; %bb.0:
@@ -26453,26 +26989,24 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v20, s34, 0
-; VI-NEXT: v_writelane_b32 v20, s35, 1
-; VI-NEXT: v_writelane_b32 v20, s36, 2
-; VI-NEXT: v_writelane_b32 v20, s37, 3
-; VI-NEXT: v_writelane_b32 v20, s38, 4
-; VI-NEXT: v_writelane_b32 v20, s39, 5
-; VI-NEXT: v_writelane_b32 v20, s48, 6
-; VI-NEXT: v_writelane_b32 v20, s49, 7
-; VI-NEXT: v_writelane_b32 v20, s50, 8
-; VI-NEXT: v_writelane_b32 v20, s51, 9
-; VI-NEXT: v_writelane_b32 v20, s52, 10
-; VI-NEXT: v_writelane_b32 v20, s53, 11
-; VI-NEXT: v_writelane_b32 v20, s54, 12
-; VI-NEXT: v_writelane_b32 v20, s55, 13
-; VI-NEXT: v_writelane_b32 v20, s64, 14
-; VI-NEXT: v_writelane_b32 v20, s65, 15
-; VI-NEXT: v_writelane_b32 v20, s66, 16
-; VI-NEXT: v_writelane_b32 v20, s67, 17
-; VI-NEXT: v_writelane_b32 v20, s30, 18
-; VI-NEXT: v_writelane_b32 v20, s31, 19
+; VI-NEXT: v_writelane_b32 v20, s36, 0
+; VI-NEXT: v_writelane_b32 v20, s37, 1
+; VI-NEXT: v_writelane_b32 v20, s38, 2
+; VI-NEXT: v_writelane_b32 v20, s39, 3
+; VI-NEXT: v_writelane_b32 v20, s48, 4
+; VI-NEXT: v_writelane_b32 v20, s49, 5
+; VI-NEXT: v_writelane_b32 v20, s50, 6
+; VI-NEXT: v_writelane_b32 v20, s51, 7
+; VI-NEXT: v_writelane_b32 v20, s52, 8
+; VI-NEXT: v_writelane_b32 v20, s53, 9
+; VI-NEXT: v_writelane_b32 v20, s54, 10
+; VI-NEXT: v_writelane_b32 v20, s55, 11
+; VI-NEXT: v_writelane_b32 v20, s64, 12
+; VI-NEXT: v_writelane_b32 v20, s65, 13
+; VI-NEXT: v_writelane_b32 v20, s66, 14
+; VI-NEXT: v_writelane_b32 v20, s67, 15
+; VI-NEXT: v_writelane_b32 v20, s30, 16
+; VI-NEXT: v_writelane_b32 v20, s31, 17
; VI-NEXT: v_readfirstlane_b32 s7, v5
; VI-NEXT: v_readfirstlane_b32 s9, v4
; VI-NEXT: v_readfirstlane_b32 s12, v3
@@ -26489,10 +27023,10 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s12, 16
@@ -26501,19 +27035,19 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -26563,26 +27097,35 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -26665,8 +27208,8 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s55, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v20, 18
+; VI-NEXT: .LBB51_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v20, 16
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -26687,33 +27230,28 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v17, s53
; VI-NEXT: v_mov_b32_e32 v18, s54
; VI-NEXT: v_mov_b32_e32 v19, s55
-; VI-NEXT: v_readlane_b32 s31, v20, 19
-; VI-NEXT: v_readlane_b32 s67, v20, 17
-; VI-NEXT: v_readlane_b32 s66, v20, 16
-; VI-NEXT: v_readlane_b32 s65, v20, 15
-; VI-NEXT: v_readlane_b32 s64, v20, 14
-; VI-NEXT: v_readlane_b32 s55, v20, 13
-; VI-NEXT: v_readlane_b32 s54, v20, 12
-; VI-NEXT: v_readlane_b32 s53, v20, 11
-; VI-NEXT: v_readlane_b32 s52, v20, 10
-; VI-NEXT: v_readlane_b32 s51, v20, 9
-; VI-NEXT: v_readlane_b32 s50, v20, 8
-; VI-NEXT: v_readlane_b32 s49, v20, 7
-; VI-NEXT: v_readlane_b32 s48, v20, 6
-; VI-NEXT: v_readlane_b32 s39, v20, 5
-; VI-NEXT: v_readlane_b32 s38, v20, 4
-; VI-NEXT: v_readlane_b32 s37, v20, 3
-; VI-NEXT: v_readlane_b32 s36, v20, 2
-; VI-NEXT: v_readlane_b32 s35, v20, 1
-; VI-NEXT: v_readlane_b32 s34, v20, 0
+; VI-NEXT: v_readlane_b32 s31, v20, 17
+; VI-NEXT: v_readlane_b32 s67, v20, 15
+; VI-NEXT: v_readlane_b32 s66, v20, 14
+; VI-NEXT: v_readlane_b32 s65, v20, 13
+; VI-NEXT: v_readlane_b32 s64, v20, 12
+; VI-NEXT: v_readlane_b32 s55, v20, 11
+; VI-NEXT: v_readlane_b32 s54, v20, 10
+; VI-NEXT: v_readlane_b32 s53, v20, 9
+; VI-NEXT: v_readlane_b32 s52, v20, 8
+; VI-NEXT: v_readlane_b32 s51, v20, 7
+; VI-NEXT: v_readlane_b32 s50, v20, 6
+; VI-NEXT: v_readlane_b32 s49, v20, 5
+; VI-NEXT: v_readlane_b32 s48, v20, 4
+; VI-NEXT: v_readlane_b32 s39, v20, 3
+; VI-NEXT: v_readlane_b32 s38, v20, 2
+; VI-NEXT: v_readlane_b32 s37, v20, 1
+; VI-NEXT: v_readlane_b32 s36, v20, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v40i16_to_v10f64_scalar:
; GFX9: ; %bb.0:
@@ -26781,10 +27319,18 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -26805,10 +27351,8 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v17, s53, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v18, s54, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -26841,7 +27385,7 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -26908,11 +27452,16 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -26934,8 +27483,6 @@ define inreg <10 x double> @bitcast_v40i16_to_v10f64_scalar(<40 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v18, s18, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v19, s19, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -27511,7 +28058,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s89, s9, 16
; SI-NEXT: s_lshr_b32 s88, s7, 16
@@ -27533,8 +28080,36 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[56:57], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[58:59], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[60:61], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[62:63], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[62:63], -1
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr73
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr75
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[62:63], s[62:63], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; SI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -27565,30 +28140,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr73
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -27629,7 +28182,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v22, s14
; SI-NEXT: v_mov_b32_e32 v21, s12
; SI-NEXT: v_mov_b32_e32 v20, s10
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
@@ -27703,7 +28256,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s12, s9, 16
; VI-NEXT: s_lshr_b32 s46, s8, 16
@@ -27725,8 +28278,36 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; VI-NEXT: s_lshr_b32 s62, s18, 16
; VI-NEXT: s_lshr_b32 s45, s17, 16
; VI-NEXT: s_lshr_b32 s63, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[10:11], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[10:11], -1
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; VI-NEXT: s_cselect_b32 s10, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s10, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -27757,30 +28338,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -27821,7 +28380,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v26, s14
; VI-NEXT: v_mov_b32_e32 v24, s13
; VI-NEXT: v_mov_b32_e32 v22, s12
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
; VI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; VI-NEXT: v_lshlrev_b32_e32 v33, 16, v33
@@ -27875,7 +28434,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s12, s9, 16
; GFX9-NEXT: s_lshr_b32 s46, s8, 16
@@ -27897,8 +28456,36 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s62, s18, 16
; GFX9-NEXT: s_lshr_b32 s45, s17, 16
; GFX9-NEXT: s_lshr_b32 s63, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[10:11], -1
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s10, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
@@ -27929,30 +28516,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -27993,7 +28558,7 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v26, s14
; GFX9-NEXT: v_mov_b32_e32 v24, s13
; GFX9-NEXT: v_mov_b32_e32 v22, s12
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -28042,13 +28607,13 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s29, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s28, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s27, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s26, 16
@@ -28066,9 +28631,36 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[14:15], s[26:27], 1.0
@@ -28099,30 +28691,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -28141,9 +28711,9 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s45 :: v_dual_mov_b32 v30, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s44 :: v_dual_mov_b32 v28, s10
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s43 :: v_dual_mov_b32 v26, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s42 :: v_dual_mov_b32 v24, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s40 :: v_dual_mov_b32 v22, s7
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s42 :: v_dual_mov_b32 v24, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, s40 :: v_dual_mov_b32 v22, s6
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v37.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v39.l
@@ -28173,13 +28743,13 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s29, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s28, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s27, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s26, 16
@@ -28197,9 +28767,36 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s8, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[28:29], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[26:27], 1.0
@@ -28230,30 +28827,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v21, s16
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s20 :: v_dual_mov_b32 v12, s22
@@ -28273,8 +28848,8 @@ define inreg <40 x half> @bitcast_v10f64_to_v40f16_scalar(<10 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s15 :: v_dual_mov_b32 v33, s12
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s13 :: v_dual_mov_b32 v31, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s10 :: v_dual_mov_b32 v27, s9
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s8 :: v_dual_mov_b32 v25, s7
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v25, s6
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v21, 0xffff, v21
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v0
@@ -29318,7 +29893,7 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; SI-NEXT: s_lshr_b32 s74, s73, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -29380,8 +29955,17 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s55, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s95
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
@@ -29542,11 +30126,8 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v18, v19, v18
; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v20
; SI-NEXT: v_or_b32_e32 v19, v21, v19
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -29579,7 +30160,7 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
+; SI-NEXT: .LBB55_6: ; %end
; SI-NEXT: v_readlane_b32 s67, v32, 15
; SI-NEXT: v_readlane_b32 s66, v32, 14
; SI-NEXT: v_readlane_b32 s65, v32, 13
@@ -29608,26 +30189,24 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v32, s34, 0
-; VI-NEXT: v_writelane_b32 v32, s35, 1
-; VI-NEXT: v_writelane_b32 v32, s36, 2
-; VI-NEXT: v_writelane_b32 v32, s37, 3
-; VI-NEXT: v_writelane_b32 v32, s38, 4
-; VI-NEXT: v_writelane_b32 v32, s39, 5
-; VI-NEXT: v_writelane_b32 v32, s48, 6
-; VI-NEXT: v_writelane_b32 v32, s49, 7
-; VI-NEXT: v_writelane_b32 v32, s50, 8
-; VI-NEXT: v_writelane_b32 v32, s51, 9
-; VI-NEXT: v_writelane_b32 v32, s52, 10
-; VI-NEXT: v_writelane_b32 v32, s53, 11
-; VI-NEXT: v_writelane_b32 v32, s54, 12
-; VI-NEXT: v_writelane_b32 v32, s55, 13
-; VI-NEXT: v_writelane_b32 v32, s64, 14
-; VI-NEXT: v_writelane_b32 v32, s65, 15
-; VI-NEXT: v_writelane_b32 v32, s66, 16
-; VI-NEXT: v_writelane_b32 v32, s67, 17
-; VI-NEXT: v_writelane_b32 v32, s30, 18
-; VI-NEXT: v_writelane_b32 v32, s31, 19
+; VI-NEXT: v_writelane_b32 v32, s36, 0
+; VI-NEXT: v_writelane_b32 v32, s37, 1
+; VI-NEXT: v_writelane_b32 v32, s38, 2
+; VI-NEXT: v_writelane_b32 v32, s39, 3
+; VI-NEXT: v_writelane_b32 v32, s48, 4
+; VI-NEXT: v_writelane_b32 v32, s49, 5
+; VI-NEXT: v_writelane_b32 v32, s50, 6
+; VI-NEXT: v_writelane_b32 v32, s51, 7
+; VI-NEXT: v_writelane_b32 v32, s52, 8
+; VI-NEXT: v_writelane_b32 v32, s53, 9
+; VI-NEXT: v_writelane_b32 v32, s54, 10
+; VI-NEXT: v_writelane_b32 v32, s55, 11
+; VI-NEXT: v_writelane_b32 v32, s64, 12
+; VI-NEXT: v_writelane_b32 v32, s65, 13
+; VI-NEXT: v_writelane_b32 v32, s66, 14
+; VI-NEXT: v_writelane_b32 v32, s67, 15
+; VI-NEXT: v_writelane_b32 v32, s30, 16
+; VI-NEXT: v_writelane_b32 v32, s31, 17
; VI-NEXT: v_readfirstlane_b32 s6, v5
; VI-NEXT: v_readfirstlane_b32 s8, v4
; VI-NEXT: v_readfirstlane_b32 s11, v3
@@ -29644,10 +30223,10 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s12, s11, 16
@@ -29656,19 +30235,19 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -29718,22 +30297,31 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v19, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s35
-; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v0, s31
+; VI-NEXT: v_mov_b32_e32 v2, s30
; VI-NEXT: v_add_f16_sdwa v0, v0, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v19
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v19
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s31
+; VI-NEXT: v_mov_b32_e32 v2, vcc_hi
; VI-NEXT: v_add_f16_sdwa v2, v2, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v19
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s30
+; VI-NEXT: v_mov_b32_e32 v3, vcc_lo
; VI-NEXT: v_add_f16_sdwa v3, v3, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v19
; VI-NEXT: v_or_b32_e32 v3, v4, v3
@@ -29801,11 +30389,8 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v20, v20, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s6, v19
; VI-NEXT: v_or_b32_e32 v19, v19, v20
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -29838,27 +30423,25 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 18
-; VI-NEXT: v_readlane_b32 s31, v32, 19
-; VI-NEXT: v_readlane_b32 s67, v32, 17
-; VI-NEXT: v_readlane_b32 s66, v32, 16
-; VI-NEXT: v_readlane_b32 s65, v32, 15
-; VI-NEXT: v_readlane_b32 s64, v32, 14
-; VI-NEXT: v_readlane_b32 s55, v32, 13
-; VI-NEXT: v_readlane_b32 s54, v32, 12
-; VI-NEXT: v_readlane_b32 s53, v32, 11
-; VI-NEXT: v_readlane_b32 s52, v32, 10
-; VI-NEXT: v_readlane_b32 s51, v32, 9
-; VI-NEXT: v_readlane_b32 s50, v32, 8
-; VI-NEXT: v_readlane_b32 s49, v32, 7
-; VI-NEXT: v_readlane_b32 s48, v32, 6
-; VI-NEXT: v_readlane_b32 s39, v32, 5
-; VI-NEXT: v_readlane_b32 s38, v32, 4
-; VI-NEXT: v_readlane_b32 s37, v32, 3
-; VI-NEXT: v_readlane_b32 s36, v32, 2
-; VI-NEXT: v_readlane_b32 s35, v32, 1
-; VI-NEXT: v_readlane_b32 s34, v32, 0
+; VI-NEXT: .LBB55_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 16
+; VI-NEXT: v_readlane_b32 s31, v32, 17
+; VI-NEXT: v_readlane_b32 s67, v32, 15
+; VI-NEXT: v_readlane_b32 s66, v32, 14
+; VI-NEXT: v_readlane_b32 s65, v32, 13
+; VI-NEXT: v_readlane_b32 s64, v32, 12
+; VI-NEXT: v_readlane_b32 s55, v32, 11
+; VI-NEXT: v_readlane_b32 s54, v32, 10
+; VI-NEXT: v_readlane_b32 s53, v32, 9
+; VI-NEXT: v_readlane_b32 s52, v32, 8
+; VI-NEXT: v_readlane_b32 s51, v32, 7
+; VI-NEXT: v_readlane_b32 s50, v32, 6
+; VI-NEXT: v_readlane_b32 s49, v32, 5
+; VI-NEXT: v_readlane_b32 s48, v32, 4
+; VI-NEXT: v_readlane_b32 s39, v32, 3
+; VI-NEXT: v_readlane_b32 s38, v32, 2
+; VI-NEXT: v_readlane_b32 s37, v32, 1
+; VI-NEXT: v_readlane_b32 s36, v32, 0
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -29931,10 +30514,18 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s53, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s54, s58, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s56, s57
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v19, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v19 op_sel_hi:[1,0]
@@ -29956,10 +30547,8 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v17, s53, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v18, s54, v19 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v19, s55, v19 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -29992,7 +30581,7 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -30059,11 +30648,16 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s17, s29, s41
; GFX11-NEXT: s_pack_ll_b32_b16 s18, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -30085,8 +30679,6 @@ define inreg <10 x double> @bitcast_v40f16_to_v10f64_scalar(<40 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v18, 0x200, s18 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v19, 0x200, s19 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -31018,206 +31610,232 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v20, s67, 17
; SI-NEXT: v_writelane_b32 v20, s68, 18
; SI-NEXT: v_writelane_b32 v20, s69, 19
-; SI-NEXT: v_writelane_b32 v20, s70, 20
-; SI-NEXT: v_writelane_b32 v20, s71, 21
-; SI-NEXT: v_writelane_b32 v20, s30, 22
-; SI-NEXT: v_writelane_b32 v20, s31, 23
-; SI-NEXT: v_readfirstlane_b32 s69, v5
-; SI-NEXT: v_readfirstlane_b32 s71, v4
-; SI-NEXT: v_readfirstlane_b32 s66, v3
-; SI-NEXT: v_readfirstlane_b32 s68, v2
-; SI-NEXT: v_readfirstlane_b32 s55, v1
-; SI-NEXT: v_readfirstlane_b32 s65, v0
-; SI-NEXT: s_lshr_b32 s36, s29, 16
-; SI-NEXT: s_lshr_b32 s54, s28, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s53, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s52, s24, 16
-; SI-NEXT: s_lshr_b32 s31, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s22, 16
-; SI-NEXT: s_lshr_b32 s30, s21, 16
-; SI-NEXT: s_lshr_b32 s50, s20, 16
+; SI-NEXT: v_writelane_b32 v20, s30, 20
+; SI-NEXT: v_writelane_b32 v20, s31, 21
+; SI-NEXT: v_readfirstlane_b32 s67, v5
+; SI-NEXT: v_readfirstlane_b32 s69, v4
+; SI-NEXT: v_readfirstlane_b32 s64, v3
+; SI-NEXT: v_readfirstlane_b32 s66, v2
+; SI-NEXT: v_readfirstlane_b32 s53, v1
+; SI-NEXT: v_readfirstlane_b32 s55, v0
+; SI-NEXT: s_lshr_b32 s34, s29, 16
+; SI-NEXT: s_lshr_b32 s52, s28, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s51, s26, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s50, s24, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s22, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s21, 16
+; SI-NEXT: s_lshr_b32 s48, s20, 16
; SI-NEXT: s_lshr_b32 s95, s19, 16
-; SI-NEXT: s_lshr_b32 s49, s18, 16
+; SI-NEXT: s_lshr_b32 s39, s18, 16
; SI-NEXT: s_lshr_b32 s94, s17, 16
-; SI-NEXT: s_lshr_b32 s48, s16, 16
-; SI-NEXT: s_lshr_b32 s39, s69, 16
-; SI-NEXT: s_lshr_b32 s70, s71, 16
-; SI-NEXT: s_lshr_b32 s38, s66, 16
-; SI-NEXT: s_lshr_b32 s67, s68, 16
-; SI-NEXT: s_lshr_b32 s37, s55, 16
-; SI-NEXT: s_lshr_b32 s64, s65, 16
+; SI-NEXT: s_lshr_b32 s38, s16, 16
+; SI-NEXT: s_lshr_b32 s37, s67, 16
+; SI-NEXT: s_lshr_b32 s68, s69, 16
+; SI-NEXT: s_lshr_b32 s36, s64, 16
+; SI-NEXT: s_lshr_b32 s65, s66, 16
+; SI-NEXT: s_lshr_b32 s35, s53, 16
+; SI-NEXT: s_lshr_b32 s54, s55, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s94, 16
-; SI-NEXT: s_or_b32 s47, s5, s7
+; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s56, s38, 16
+; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
; SI-NEXT: s_lshl_b32 s7, s95, 16
-; SI-NEXT: s_or_b32 s43, s5, s7
+; SI-NEXT: s_or_b32 s42, s4, s56
+; SI-NEXT: s_and_b32 s4, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s58, s39, 16
+; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s30, 16
-; SI-NEXT: s_or_b32 s41, s5, s7
+; SI-NEXT: s_lshl_b32 s7, vcc_lo, 16
+; SI-NEXT: s_or_b32 s40, s4, s58
+; SI-NEXT: s_and_b32 s4, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s60, s48, 16
+; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s31, 16
-; SI-NEXT: s_or_b32 s15, s5, s7
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 16
+; SI-NEXT: s_or_b32 s14, s4, s60
+; SI-NEXT: s_and_b32 s4, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s62, s49, 16
+; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
-; SI-NEXT: s_or_b32 s13, s5, s7
+; SI-NEXT: s_lshl_b32 s7, s30, 16
+; SI-NEXT: s_or_b32 s12, s4, s62
+; SI-NEXT: s_and_b32 s4, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s72, s50, 16
+; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s35, 16
-; SI-NEXT: s_or_b32 s11, s5, s7
+; SI-NEXT: s_lshl_b32 s7, s31, 16
+; SI-NEXT: s_or_b32 s10, s4, s72
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s74, s51, 16
+; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s34, 16
+; SI-NEXT: s_or_b32 s8, s4, s74
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s76, s52, 16
+; SI-NEXT: s_or_b32 s77, s5, s7
+; SI-NEXT: s_and_b32 s5, s53, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s35, 16
+; SI-NEXT: s_or_b32 s6, s4, s76
+; SI-NEXT: s_and_b32 s4, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s46, s54, 16
+; SI-NEXT: s_or_b32 s47, s5, s7
+; SI-NEXT: s_and_b32 s5, s64, 0xffff
; SI-NEXT: s_lshl_b32 s7, s36, 16
-; SI-NEXT: s_or_b32 s9, s5, s7
-; SI-NEXT: s_and_b32 s5, s55, 0xffff
+; SI-NEXT: s_or_b32 s4, s4, s46
+; SI-NEXT: s_lshl_b32 s44, s65, 16
+; SI-NEXT: s_or_b32 s45, s5, s7
+; SI-NEXT: s_and_b32 s5, s67, 0xffff
; SI-NEXT: s_lshl_b32 s7, s37, 16
-; SI-NEXT: s_or_b32 s7, s5, s7
-; SI-NEXT: s_and_b32 s5, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s45, s38, 16
-; SI-NEXT: s_and_b32 s44, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s46, s48, 16
-; SI-NEXT: s_or_b32 s5, s5, s45
-; SI-NEXT: s_and_b32 s45, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s39, 16
-; SI-NEXT: s_lshl_b32 s42, s49, 16
-; SI-NEXT: s_or_b32 s91, s45, s56
-; SI-NEXT: s_or_b32 s44, s44, s46
-; SI-NEXT: s_lshr_b64 s[56:57], s[46:47], 16
-; SI-NEXT: s_and_b32 s46, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s50, 16
-; SI-NEXT: s_or_b32 s46, s46, s42
-; SI-NEXT: s_lshr_b64 s[58:59], s[42:43], 16
-; SI-NEXT: s_and_b32 s42, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s51, 16
-; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_lshr_b64 s[60:61], s[40:41], 16
-; SI-NEXT: s_and_b32 s40, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s52, 16
-; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_lshr_b64 s[62:63], s[14:15], 16
-; SI-NEXT: s_and_b32 s14, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s53, 16
-; SI-NEXT: s_or_b32 s14, s14, s12
-; SI-NEXT: s_lshr_b64 s[72:73], s[12:13], 16
-; SI-NEXT: s_and_b32 s12, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s54, 16
-; SI-NEXT: s_or_b32 s12, s12, s10
-; SI-NEXT: s_lshr_b64 s[74:75], s[10:11], 16
-; SI-NEXT: s_and_b32 s10, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s64, 16
-; SI-NEXT: s_or_b32 s10, s10, s8
-; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
-; SI-NEXT: s_and_b32 s8, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s4, s67, 16
-; SI-NEXT: s_or_b32 s8, s8, s6
-; SI-NEXT: s_lshr_b64 s[78:79], s[6:7], 16
-; SI-NEXT: s_and_b32 s6, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s90, s70, 16
-; SI-NEXT: s_or_b32 s6, s6, s4
-; SI-NEXT: s_lshr_b64 s[88:89], s[4:5], 16
-; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_mov_b32 s45, s47
-; SI-NEXT: s_mov_b32 s47, s43
-; SI-NEXT: s_mov_b32 s43, s41
-; SI-NEXT: s_mov_b32 s41, s15
-; SI-NEXT: s_mov_b32 s15, s13
-; SI-NEXT: s_mov_b32 s13, s11
-; SI-NEXT: s_mov_b32 s11, s9
-; SI-NEXT: s_mov_b32 s9, s7
-; SI-NEXT: s_mov_b32 s7, s5
-; SI-NEXT: s_or_b32 s4, s4, s90
-; SI-NEXT: s_mov_b32 s5, s91
+; SI-NEXT: s_lshr_b64 s[78:79], s[46:47], 16
+; SI-NEXT: s_and_b32 s46, s66, 0xffff
+; SI-NEXT: s_or_b32 s91, s5, s7
+; SI-NEXT: s_lshl_b32 s90, s68, 16
+; SI-NEXT: s_or_b32 s46, s46, s44
+; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
+; SI-NEXT: s_and_b32 s44, s69, 0xffff
+; SI-NEXT: s_mov_b32 s43, s57
+; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
+; SI-NEXT: s_mov_b32 s41, s59
+; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
+; SI-NEXT: s_mov_b32 s15, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_mov_b32 s13, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
+; SI-NEXT: s_mov_b32 s11, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
+; SI-NEXT: s_mov_b32 s9, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
+; SI-NEXT: s_mov_b32 s7, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[76:77], 16
+; SI-NEXT: s_mov_b32 s5, s47
+; SI-NEXT: s_mov_b32 s47, s45
+; SI-NEXT: s_or_b32 s44, s44, s90
+; SI-NEXT: s_mov_b32 s45, s91
; SI-NEXT: s_lshr_b64 s[90:91], s[90:91], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
-; SI-NEXT: s_add_i32 s71, s71, 3
-; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_mov_b64 s[92:93], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[92:93], -1
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[92:93], s[92:93], exec
+; SI-NEXT: s_cselect_b32 s57, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s57, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_and_b32 s4, s69, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s67, s67, 3
+; SI-NEXT: s_add_i32 s44, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s67, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s37, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s39, 16
-; SI-NEXT: s_add_i32 s68, s68, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s67, 16
; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s38, 16
-; SI-NEXT: s_add_i32 s65, s65, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s64, 16
+; SI-NEXT: s_add_i32 s45, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s66, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s65, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s64, s64, 3
+; SI-NEXT: s_add_i32 s46, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s64, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s36, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s55, s55, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s55, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s37, 16
+; SI-NEXT: s_add_i32 s47, s4, 0x30000
+; SI-NEXT: s_and_b32 s4, s55, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s54, 16
+; SI-NEXT: s_add_i32 s53, s53, 3
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s53, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s35, 16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s54, 16
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s52, 16
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s36, 16
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s34, 16
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s53, 16
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s51, 16
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s35, 16
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s31, 16
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_or_b32 s13, s14, s13
-; SI-NEXT: s_and_b32 s14, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s52, 16
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s10, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s50, 16
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_or_b32 s14, s15, s14
-; SI-NEXT: s_and_b32 s15, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s34, 16
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s30, 16
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s15, s24, s15
-; SI-NEXT: s_and_b32 s22, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s51, 16
-; SI-NEXT: s_or_b32 s22, s24, s22
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s12, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s49, 16
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_add_i32 s40, s22, 0x30000
-; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s31, 16
-; SI-NEXT: s_or_b32 s22, s23, s22
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_and_b32 s13, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s14, vcc_hi, 16
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_add_i32 s41, s22, 0x30000
-; SI-NEXT: s_and_b32 s20, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s50, 16
-; SI-NEXT: s_or_b32 s20, s22, s20
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_and_b32 s14, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s48, 16
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_add_i32 s42, s20, 0x30000
-; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s30, 16
-; SI-NEXT: s_or_b32 s20, s21, s20
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_and_b32 s15, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s20, vcc_lo, 16
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_add_i32 s43, s20, 0x30000
+; SI-NEXT: s_or_b32 s15, s20, s15
; SI-NEXT: s_and_b32 s18, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s49, 16
+; SI-NEXT: s_lshl_b32 s20, s39, 16
; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_add_i32 s46, s18, 0x30000
+; SI-NEXT: s_add_i32 s40, s18, 0x30000
; SI-NEXT: s_and_b32 s18, s19, 0xffff
; SI-NEXT: s_lshl_b32 s19, s95, 16
; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_add_i32 s47, s18, 0x30000
+; SI-NEXT: s_add_i32 s41, s18, 0x30000
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s48, 16
+; SI-NEXT: s_lshl_b32 s18, s38, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_add_i32 s44, s16, 0x30000
+; SI-NEXT: s_add_i32 s42, s16, 0x30000
; SI-NEXT: s_and_b32 s16, s17, 0xffff
; SI-NEXT: s_lshl_b32 s17, s94, 16
; SI-NEXT: s_or_b32 s16, s17, s16
@@ -31233,112 +31851,110 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s13, s13, 0x30000
; SI-NEXT: s_add_i32 s14, s14, 0x30000
; SI-NEXT: s_add_i32 s15, s15, 0x30000
-; SI-NEXT: s_add_i32 s45, s16, 0x30000
-; SI-NEXT: s_lshr_b64 s[56:57], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[58:59], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[60:61], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[72:73], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[88:89], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[4:5], 16
-; SI-NEXT: s_lshr_b32 s94, s45, 16
-; SI-NEXT: s_lshr_b32 s95, s47, 16
-; SI-NEXT: s_lshr_b32 s30, s43, 16
-; SI-NEXT: s_lshr_b32 s31, s41, 16
-; SI-NEXT: s_lshr_b32 s34, s15, 16
-; SI-NEXT: s_lshr_b32 s35, s13, 16
-; SI-NEXT: s_lshr_b32 s36, s11, 16
-; SI-NEXT: s_lshr_b32 s37, s9, 16
-; SI-NEXT: s_lshr_b32 s38, s7, 16
-; SI-NEXT: s_lshr_b32 s39, s5, 16
-; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: s_and_b32 s16, s44, 0xffff
+; SI-NEXT: s_add_i32 s43, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[56:57], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[44:45], 16
+; SI-NEXT: s_lshr_b32 s94, s43, 16
+; SI-NEXT: s_lshr_b32 s95, s41, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s15, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s13, 16
+; SI-NEXT: s_lshr_b32 s30, s11, 16
+; SI-NEXT: s_lshr_b32 s31, s9, 16
+; SI-NEXT: s_lshr_b32 s34, s7, 16
+; SI-NEXT: s_lshr_b32 s35, s5, 16
+; SI-NEXT: s_lshr_b32 s36, s47, 16
+; SI-NEXT: s_lshr_b32 s37, s45, 16
+; SI-NEXT: .LBB57_5: ; %end
+; SI-NEXT: s_and_b32 s16, s42, 0xffff
; SI-NEXT: s_lshl_b32 s17, s56, 16
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s45, 0xffff
+; SI-NEXT: s_and_b32 s17, s43, 0xffff
; SI-NEXT: s_lshl_b32 s18, s94, 16
; SI-NEXT: s_or_b32 s17, s17, s18
-; SI-NEXT: s_and_b32 s18, s46, 0xffff
+; SI-NEXT: s_and_b32 s18, s40, 0xffff
; SI-NEXT: s_lshl_b32 s19, s58, 16
; SI-NEXT: s_or_b32 s18, s18, s19
-; SI-NEXT: s_and_b32 s19, s47, 0xffff
+; SI-NEXT: s_and_b32 s19, s41, 0xffff
; SI-NEXT: s_lshl_b32 s20, s95, 16
; SI-NEXT: s_or_b32 s19, s19, s20
-; SI-NEXT: s_and_b32 s20, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s60, 16
-; SI-NEXT: s_or_b32 s20, s20, s21
-; SI-NEXT: s_and_b32 s21, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s30, 16
-; SI-NEXT: s_or_b32 s21, s21, s22
-; SI-NEXT: s_and_b32 s22, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s62, 16
-; SI-NEXT: s_or_b32 s22, s22, s23
-; SI-NEXT: s_and_b32 s23, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s31, 16
-; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s72, 16
-; SI-NEXT: s_or_b32 s14, s14, s24
+; SI-NEXT: s_lshl_b32 s20, s60, 16
+; SI-NEXT: s_or_b32 s14, s14, s20
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s34, 16
-; SI-NEXT: s_or_b32 s15, s15, s24
+; SI-NEXT: s_lshl_b32 s20, vcc_lo, 16
+; SI-NEXT: s_or_b32 s15, s15, s20
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s74, 16
-; SI-NEXT: s_or_b32 s12, s12, s24
+; SI-NEXT: s_lshl_b32 s20, s62, 16
+; SI-NEXT: s_or_b32 s12, s12, s20
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s35, 16
-; SI-NEXT: s_or_b32 s13, s13, s24
+; SI-NEXT: s_lshl_b32 s20, vcc_hi, 16
+; SI-NEXT: s_or_b32 s13, s13, s20
; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s76, 16
-; SI-NEXT: s_or_b32 s10, s10, s24
+; SI-NEXT: s_lshl_b32 s20, s72, 16
+; SI-NEXT: s_or_b32 s10, s10, s20
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s36, 16
-; SI-NEXT: s_or_b32 s11, s11, s24
+; SI-NEXT: s_lshl_b32 s20, s30, 16
+; SI-NEXT: s_or_b32 s11, s11, s20
; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s78, 16
-; SI-NEXT: s_or_b32 s8, s8, s24
+; SI-NEXT: s_lshl_b32 s20, s74, 16
+; SI-NEXT: s_or_b32 s8, s8, s20
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s37, 16
-; SI-NEXT: s_or_b32 s9, s9, s24
+; SI-NEXT: s_lshl_b32 s20, s31, 16
+; SI-NEXT: s_or_b32 s9, s9, s20
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s88, 16
-; SI-NEXT: s_or_b32 s6, s6, s24
+; SI-NEXT: s_lshl_b32 s20, s76, 16
+; SI-NEXT: s_or_b32 s6, s6, s20
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s38, 16
-; SI-NEXT: s_or_b32 s7, s7, s24
+; SI-NEXT: s_lshl_b32 s20, s34, 16
+; SI-NEXT: s_or_b32 s7, s7, s20
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s90, 16
-; SI-NEXT: s_or_b32 s4, s4, s24
+; SI-NEXT: s_lshl_b32 s20, s78, 16
+; SI-NEXT: s_or_b32 s4, s4, s20
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s39, 16
-; SI-NEXT: s_or_b32 s5, s5, s24
-; SI-NEXT: v_readlane_b32 s30, v20, 22
+; SI-NEXT: s_lshl_b32 s20, s35, 16
+; SI-NEXT: s_or_b32 s5, s5, s20
+; SI-NEXT: s_and_b32 s20, s46, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s88, 16
+; SI-NEXT: s_or_b32 s20, s20, s21
+; SI-NEXT: s_and_b32 s21, s47, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s36, 16
+; SI-NEXT: s_or_b32 s21, s21, s22
+; SI-NEXT: s_and_b32 s22, s44, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: s_or_b32 s22, s22, s23
+; SI-NEXT: s_and_b32 s23, s45, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s37, 16
+; SI-NEXT: s_or_b32 s23, s23, s24
+; SI-NEXT: v_readlane_b32 s30, v20, 20
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v3, s19
-; SI-NEXT: v_mov_b32_e32 v4, s20
-; SI-NEXT: v_mov_b32_e32 v5, s21
-; SI-NEXT: v_mov_b32_e32 v6, s22
-; SI-NEXT: v_mov_b32_e32 v7, s23
-; SI-NEXT: v_mov_b32_e32 v8, s14
-; SI-NEXT: v_mov_b32_e32 v9, s15
-; SI-NEXT: v_mov_b32_e32 v10, s12
-; SI-NEXT: v_mov_b32_e32 v11, s13
-; SI-NEXT: v_mov_b32_e32 v12, s10
-; SI-NEXT: v_mov_b32_e32 v13, s11
-; SI-NEXT: v_mov_b32_e32 v14, s8
-; SI-NEXT: v_mov_b32_e32 v15, s9
-; SI-NEXT: v_mov_b32_e32 v16, s6
-; SI-NEXT: v_mov_b32_e32 v17, s7
-; SI-NEXT: v_mov_b32_e32 v18, s4
-; SI-NEXT: v_mov_b32_e32 v19, s5
-; SI-NEXT: v_readlane_b32 s31, v20, 23
-; SI-NEXT: v_readlane_b32 s71, v20, 21
-; SI-NEXT: v_readlane_b32 s70, v20, 20
+; SI-NEXT: v_mov_b32_e32 v4, s14
+; SI-NEXT: v_mov_b32_e32 v5, s15
+; SI-NEXT: v_mov_b32_e32 v6, s12
+; SI-NEXT: v_mov_b32_e32 v7, s13
+; SI-NEXT: v_mov_b32_e32 v8, s10
+; SI-NEXT: v_mov_b32_e32 v9, s11
+; SI-NEXT: v_mov_b32_e32 v10, s8
+; SI-NEXT: v_mov_b32_e32 v11, s9
+; SI-NEXT: v_mov_b32_e32 v12, s6
+; SI-NEXT: v_mov_b32_e32 v13, s7
+; SI-NEXT: v_mov_b32_e32 v14, s4
+; SI-NEXT: v_mov_b32_e32 v15, s5
+; SI-NEXT: v_mov_b32_e32 v16, s20
+; SI-NEXT: v_mov_b32_e32 v17, s21
+; SI-NEXT: v_mov_b32_e32 v18, s22
+; SI-NEXT: v_mov_b32_e32 v19, s23
+; SI-NEXT: v_readlane_b32 s31, v20, 21
; SI-NEXT: v_readlane_b32 s69, v20, 19
; SI-NEXT: v_readlane_b32 s68, v20, 18
; SI-NEXT: v_readlane_b32 s67, v20, 17
@@ -31364,28 +31980,6 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v40i16_to_v40f16_scalar:
; VI: ; %bb.0:
@@ -31418,10 +32012,18 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s47, s46, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s63, s63, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -31462,7 +32064,7 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s10, s10, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -31544,8 +32146,6 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v18, s9
; VI-NEXT: v_mov_b32_e32 v19, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v40i16_to_v40f16_scalar:
; GFX9: ; %bb.0:
@@ -31578,10 +32178,18 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s58, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v6
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s63, s57
; GFX9-NEXT: v_pk_add_u16 v19, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s62, s56
@@ -31642,10 +32250,8 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v17
; GFX9-NEXT: v_lshrrev_b32_e32 v21, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v19, s63
; GFX9-NEXT: v_mov_b32_e32 v18, s62
; GFX9-NEXT: v_mov_b32_e32 v17, s61
@@ -31686,7 +32292,7 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v37, s8
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -31757,11 +32363,16 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s56, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s58, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s57, s47
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s46, s56, s46
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
@@ -31823,8 +32434,6 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v19
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s57 :: v_dual_mov_b32 v18, s56
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v16, s28
@@ -31899,11 +32508,16 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s56, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s58, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s58, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s57, s47
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s46, s56, s46
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
@@ -31965,8 +32579,6 @@ define inreg <40 x half> @bitcast_v40i16_to_v40f16_scalar(<40 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 16, v16
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v20, 16, v15
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s57 :: v_dual_mov_b32 v16, s56
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v18, s28
@@ -32748,10 +33360,18 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s56, s8, 16
; SI-NEXT: v_readfirstlane_b32 s4, v6
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s63
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s62
@@ -32923,10 +33543,8 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[24:25], v[14:15], 16
; SI-NEXT: v_lshr_b64 v[22:23], v[16:17], 16
; SI-NEXT: v_lshr_b64 v[20:21], v[18:19], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v50, s58
; SI-NEXT: v_mov_b32_e32 v40, s43
; SI-NEXT: v_mov_b32_e32 v51, s47
@@ -32971,7 +33589,7 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v24, s56
; SI-NEXT: v_mov_b32_e32 v22, s45
; SI-NEXT: v_mov_b32_e32 v20, s44
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v38
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v48
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -33078,10 +33696,18 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s63, s62, 16
; VI-NEXT: v_readfirstlane_b32 s4, v6
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v20, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v20
; VI-NEXT: v_add_f16_e32 v39, s43, v20
@@ -33123,10 +33749,8 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v21, s47, v20
; VI-NEXT: v_add_f16_e32 v19, s44, v20
; VI-NEXT: v_add_f16_e32 v20, s45, v20
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v20, s45
; VI-NEXT: v_mov_b32_e32 v19, s44
; VI-NEXT: v_mov_b32_e32 v21, s47
@@ -33167,7 +33791,7 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v39, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; VI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -33241,10 +33865,18 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s58, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v6
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s63, s57
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v19, s4, v0 op_sel_hi:[1,0]
@@ -33306,10 +33938,8 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v17
; GFX9-NEXT: v_lshrrev_b32_e32 v21, 16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v19, s63
; GFX9-NEXT: v_mov_b32_e32 v18, s62
; GFX9-NEXT: v_mov_b32_e32 v17, s61
@@ -33350,7 +33980,7 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v37, s8
; GFX9-NEXT: v_mov_b32_e32 v38, s7
; GFX9-NEXT: v_mov_b32_e32 v39, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -33421,11 +34051,16 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s56, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s58, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s58, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s57, s47
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s46, s56, s46
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
@@ -33487,8 +34122,6 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v19
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s57 :: v_dual_mov_b32 v18, s56
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v16, s28
@@ -33563,11 +34196,16 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s56, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s58, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s58
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s58, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s58, s58, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s58, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s58, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s57, s47
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s46, s56, s46
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s29, s29, s45
@@ -33629,8 +34267,6 @@ define inreg <40 x i16> @bitcast_v40f16_to_v40i16_scalar(<40 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v21, 16, v16
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v20, 16, v15
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, s57 :: v_dual_mov_b32 v16, s56
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v18, s28
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 77c047f492cac..4443a087a2dd3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -86,71 +86,92 @@ define inreg double @bitcast_i64_to_f64_scalar(i64 inreg %a, i32 inreg %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_i64_to_f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_i64_to_f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_i64_to_f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -244,15 +265,21 @@ define inreg i64 @bitcast_f64_to_i64_scalar(double inreg %a, i32 inreg %b) #0 {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -261,15 +288,21 @@ define inreg i64 @bitcast_f64_to_i64_scalar(double inreg %a, i32 inreg %b) #0 {
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -278,15 +311,21 @@ define inreg i64 @bitcast_f64_to_i64_scalar(double inreg %a, i32 inreg %b) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -296,15 +335,18 @@ define inreg i64 @bitcast_f64_to_i64_scalar(double inreg %a, i32 inreg %b) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -405,71 +447,92 @@ define inreg <2 x i32> @bitcast_i64_to_v2i32_scalar(i64 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_i64_to_v2i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_i64_to_v2i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_i64_to_v2i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -566,71 +629,92 @@ define inreg i64 @bitcast_v2i32_to_i64_scalar(<2 x i32> inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v2i32_to_i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v2i32_to_i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v2i32_to_i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -728,71 +812,92 @@ define inreg <2 x float> @bitcast_i64_to_v2f32_scalar(i64 inreg %a, i32 inreg %b
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_i64_to_v2f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_i64_to_v2f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_i64_to_v2f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -888,16 +993,22 @@ define inreg i64 @bitcast_v2f32_to_i64_scalar(<2 x float> inreg %a, i32 inreg %b
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -906,16 +1017,22 @@ define inreg i64 @bitcast_v2f32_to_i64_scalar(<2 x float> inreg %a, i32 inreg %b
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -924,16 +1041,22 @@ define inreg i64 @bitcast_v2f32_to_i64_scalar(<2 x float> inreg %a, i32 inreg %b
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -943,16 +1066,19 @@ define inreg i64 @bitcast_v2f32_to_i64_scalar(<2 x float> inreg %a, i32 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1067,17 +1193,27 @@ define inreg <4 x i16> @bitcast_i64_to_v4i16_scalar(i64 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -1087,63 +1223,74 @@ define inreg <4 x i16> @bitcast_i64_to_v4i16_scalar(i64 inreg %a, i32 inreg %b)
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_i64_to_v4i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_i64_to_v4i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_i64_to_v4i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1276,7 +1423,7 @@ define inreg i64 @bitcast_v4i16_to_i64_scalar(<4 x i16> inreg %a, i32 inreg %b)
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -1284,8 +1431,17 @@ define inreg i64 @bitcast_v4i16_to_i64_scalar(<4 x i16> inreg %a, i32 inreg %b)
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -1296,22 +1452,27 @@ define inreg i64 @bitcast_v4i16_to_i64_scalar(<4 x i16> inreg %a, i32 inreg %b)
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
+; SI-NEXT: .LBB15_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v4i16_to_i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -1322,27 +1483,31 @@ define inreg i64 @bitcast_v4i16_to_i64_scalar(<4 x i16> inreg %a, i32 inreg %b)
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v4i16_to_i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1352,16 +1517,19 @@ define inreg i64 @bitcast_v4i16_to_i64_scalar(<4 x i16> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1476,17 +1644,27 @@ define inreg <4 x half> @bitcast_i64_to_v4f16_scalar(i64 inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -1496,63 +1674,74 @@ define inreg <4 x half> @bitcast_i64_to_v4f16_scalar(i64 inreg %a, i32 inreg %b)
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_i64_to_v4f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_i64_to_v4f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_i64_to_v4f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1694,7 +1883,7 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -1702,8 +1891,17 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
@@ -1721,10 +1919,7 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -1733,10 +1928,18 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -1749,9 +1952,7 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1760,17 +1961,23 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1780,16 +1987,19 @@ define inreg i64 @bitcast_v4f16_to_i64_scalar(<4 x half> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1914,21 +2124,33 @@ define inreg <4 x bfloat> @bitcast_i64_to_v4bf16_scalar(i64 inreg %a, i32 inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_4
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s17, 16
; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB21_3
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s16, 3
; SI-NEXT: s_addc_u32 s5, s17, 0
; SI-NEXT: s_and_b32 s7, s5, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s4, 16
-; SI-NEXT: .LBB21_3: ; %end
+; SI-NEXT: .LBB21_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
@@ -1938,65 +2160,74 @@ define inreg <4 x bfloat> @bitcast_i64_to_v4bf16_scalar(i64 inreg %a, i32 inreg
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB21_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB21_2
;
; VI-LABEL: bitcast_i64_to_v4bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_4
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_3
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB21_3: ; %end
+; VI-NEXT: .LBB21_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB21_4:
-; VI-NEXT: s_branch .LBB21_2
;
; GFX9-LABEL: bitcast_i64_to_v4bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_3
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB21_3: ; %end
+; GFX9-NEXT: .LBB21_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB21_4:
-; GFX9-NEXT: s_branch .LBB21_2
;
; GFX11-LABEL: bitcast_i64_to_v4bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_3
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB21_3: ; %end
+; GFX11-NEXT: .LBB21_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB21_4:
-; GFX11-NEXT: s_branch .LBB21_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2292,14 +2523,23 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -2312,20 +2552,25 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v4bf16_to_i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_3
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_4
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v4, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v4
@@ -2365,9 +2610,7 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_3:
-; VI-NEXT: s_branch .LBB23_2
-; VI-NEXT: .LBB23_4:
+; VI-NEXT: .LBB23_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -2376,10 +2619,18 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_4
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -2421,9 +2672,7 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_3:
-; GFX9-NEXT: s_branch .LBB23_2
-; GFX9-NEXT: .LBB23_4:
+; GFX9-NEXT: .LBB23_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2433,11 +2682,16 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-TRUE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -2479,8 +2733,6 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB23_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB23_2
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -2490,11 +2742,16 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB23_4
-; GFX11-FAKE16-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB23_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -2540,8 +2797,6 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB23_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB23_2
; GFX11-FAKE16-NEXT: .LBB23_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -2779,7 +3034,7 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_4
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s17, 24
; SI-NEXT: s_lshr_b32 s13, s17, 16
@@ -2787,8 +3042,22 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB25_3
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_lshr_b32 s12, s17, 24
@@ -2797,7 +3066,7 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: .LBB25_3: ; %end
+; SI-NEXT: .LBB25_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2807,20 +3076,12 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; SI-NEXT: v_mov_b32_e32 v6, s13
; SI-NEXT: v_mov_b32_e32 v7, s12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB25_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB25_2
;
; VI-LABEL: bitcast_i64_to_v8i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_4
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s5, s17, 24
@@ -2828,8 +3089,22 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB25_3
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
@@ -2838,7 +3113,7 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
-; VI-NEXT: .LBB25_3: ; %end
+; VI-NEXT: .LBB25_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s11
; VI-NEXT: v_mov_b32_e32 v2, s10
@@ -2848,20 +3123,12 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v7, s5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_4:
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: s_branch .LBB25_2
;
; GFX9-LABEL: bitcast_i64_to_v8i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
@@ -2869,8 +3136,22 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB25_3
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
@@ -2879,7 +3160,7 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: .LBB25_3: ; %end
+; GFX9-NEXT: .LBB25_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s11
; GFX9-NEXT: v_mov_b32_e32 v2, s10
@@ -2889,21 +3170,13 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_4:
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: s_branch .LBB25_2
;
; GFX11-LABEL: bitcast_i64_to_v8i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
@@ -2911,9 +3184,22 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX11-NEXT: s_lshr_b32 s5, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_3
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB25_3
+; GFX11-NEXT: .LBB25_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: .LBB25_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
@@ -2922,21 +3208,13 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_lshr_b32 s5, s1, 8
; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: .LBB25_3: ; %end
+; GFX11-NEXT: .LBB25_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_4:
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB25_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3292,7 +3570,7 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -3312,8 +3590,17 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -3340,19 +3627,16 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v8i8_to_i64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_4
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -3367,8 +3651,17 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB27_3
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -3390,17 +3683,14 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB27_3: ; %end
+; VI-NEXT: .LBB27_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB27_2
;
; GFX9-LABEL: bitcast_v8i8_to_i64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -3415,8 +3705,17 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB27_3
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -3437,18 +3736,15 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB27_3: ; %end
+; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB27_2
;
; GFX11-LABEL: bitcast_v8i8_to_i64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -3462,9 +3758,17 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_3
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB27_3
+; GFX11-NEXT: .LBB27_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB27_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -3489,11 +3793,8 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB27_3: ; %end
+; GFX11-NEXT: .LBB27_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB27_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB27_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3587,15 +3888,21 @@ define inreg <2 x i32> @bitcast_f64_to_v2i32_scalar(double inreg %a, i32 inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -3604,15 +3911,21 @@ define inreg <2 x i32> @bitcast_f64_to_v2i32_scalar(double inreg %a, i32 inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3621,15 +3934,21 @@ define inreg <2 x i32> @bitcast_f64_to_v2i32_scalar(double inreg %a, i32 inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3639,15 +3958,18 @@ define inreg <2 x i32> @bitcast_f64_to_v2i32_scalar(double inreg %a, i32 inreg %
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -3747,71 +4069,92 @@ define inreg double @bitcast_v2i32_to_f64_scalar(<2 x i32> inreg %a, i32 inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB31_3: ; %end
+; SI-NEXT: .LBB31_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v2i32_to_f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v2i32_to_f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_3
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB31_3: ; %end
+; GFX9-NEXT: .LBB31_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_4:
-; GFX9-NEXT: s_branch .LBB31_2
;
; GFX11-LABEL: bitcast_v2i32_to_f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_3
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB31_3: ; %end
+; GFX11-NEXT: .LBB31_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_4:
-; GFX11-NEXT: s_branch .LBB31_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3905,15 +4248,21 @@ define inreg <2 x float> @bitcast_f64_to_v2f32_scalar(double inreg %a, i32 inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -3922,15 +4271,21 @@ define inreg <2 x float> @bitcast_f64_to_v2f32_scalar(double inreg %a, i32 inreg
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3939,15 +4294,21 @@ define inreg <2 x float> @bitcast_f64_to_v2f32_scalar(double inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3957,15 +4318,18 @@ define inreg <2 x float> @bitcast_f64_to_v2f32_scalar(double inreg %a, i32 inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4064,16 +4428,22 @@ define inreg double @bitcast_v2f32_to_f64_scalar(<2 x float> inreg %a, i32 inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -4082,16 +4452,22 @@ define inreg double @bitcast_v2f32_to_f64_scalar(<2 x float> inreg %a, i32 inreg
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4100,16 +4476,22 @@ define inreg double @bitcast_v2f32_to_f64_scalar(<2 x float> inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4119,16 +4501,19 @@ define inreg double @bitcast_v2f32_to_f64_scalar(<2 x float> inreg %a, i32 inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4239,26 +4624,32 @@ define inreg <4 x i16> @bitcast_f64_to_v4i16_scalar(double inreg %a, i32 inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_3
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB37_4
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: s_branch .LBB37_5
-; SI-NEXT: .LBB37_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB37_2
-; SI-NEXT: .LBB37_4:
+; SI-NEXT: s_branch .LBB37_6
+; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: .LBB37_5: ; %end
+; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -4271,15 +4662,21 @@ define inreg <4 x i16> @bitcast_f64_to_v4i16_scalar(double inreg %a, i32 inreg %
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_3
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_4
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_3:
-; VI-NEXT: s_branch .LBB37_2
-; VI-NEXT: .LBB37_4:
+; VI-NEXT: .LBB37_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4288,15 +4685,21 @@ define inreg <4 x i16> @bitcast_f64_to_v4i16_scalar(double inreg %a, i32 inreg %
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_4
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_3:
-; GFX9-NEXT: s_branch .LBB37_2
-; GFX9-NEXT: .LBB37_4:
+; GFX9-NEXT: .LBB37_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4306,15 +4709,18 @@ define inreg <4 x i16> @bitcast_f64_to_v4i16_scalar(double inreg %a, i32 inreg %
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_4
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_3:
-; GFX11-NEXT: s_branch .LBB37_2
; GFX11-NEXT: .LBB37_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4450,7 +4856,7 @@ define inreg double @bitcast_v4i16_to_f64_scalar(<4 x i16> inreg %a, i32 inreg %
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -4458,8 +4864,17 @@ define inreg double @bitcast_v4i16_to_f64_scalar(<4 x i16> inreg %a, i32 inreg %
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -4470,22 +4885,27 @@ define inreg double @bitcast_v4i16_to_f64_scalar(<4 x i16> inreg %a, i32 inreg %
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v4i16_to_f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_4
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_3
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -4496,27 +4916,31 @@ define inreg double @bitcast_v4i16_to_f64_scalar(<4 x i16> inreg %a, i32 inreg %
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB39_3: ; %end
+; VI-NEXT: .LBB39_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB39_4:
-; VI-NEXT: s_branch .LBB39_2
;
; GFX9-LABEL: bitcast_v4i16_to_f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4526,16 +4950,19 @@ define inreg double @bitcast_v4i16_to_f64_scalar(<4 x i16> inreg %a, i32 inreg %
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4646,26 +5073,32 @@ define inreg <4 x half> @bitcast_f64_to_v4f16_scalar(double inreg %a, i32 inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_3
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_4
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: s_branch .LBB41_5
-; SI-NEXT: .LBB41_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB41_2
-; SI-NEXT: .LBB41_4:
+; SI-NEXT: s_branch .LBB41_6
+; SI-NEXT: .LBB41_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: .LBB41_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -4678,15 +5111,21 @@ define inreg <4 x half> @bitcast_f64_to_v4f16_scalar(double inreg %a, i32 inreg
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_3
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB41_4
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_3:
-; VI-NEXT: s_branch .LBB41_2
-; VI-NEXT: .LBB41_4:
+; VI-NEXT: .LBB41_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4695,15 +5134,21 @@ define inreg <4 x half> @bitcast_f64_to_v4f16_scalar(double inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB41_4
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_3:
-; GFX9-NEXT: s_branch .LBB41_2
-; GFX9-NEXT: .LBB41_4:
+; GFX9-NEXT: .LBB41_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4713,15 +5158,18 @@ define inreg <4 x half> @bitcast_f64_to_v4f16_scalar(double inreg %a, i32 inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_4
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB41_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_3:
-; GFX11-NEXT: s_branch .LBB41_2
; GFX11-NEXT: .LBB41_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4866,7 +5314,7 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_3
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -4874,8 +5322,17 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB43_4
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
@@ -4893,10 +5350,7 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB43_2
-; SI-NEXT: .LBB43_4:
+; SI-NEXT: .LBB43_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -4905,10 +5359,18 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -4921,9 +5383,7 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -4932,17 +5392,23 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -4952,16 +5418,19 @@ define inreg double @bitcast_v4f16_to_f64_scalar(<4 x half> inreg %a, i32 inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -5081,32 +5550,38 @@ define inreg <4 x bfloat> @bitcast_f64_to_v4bf16_scalar(double inreg %a, i32 inr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_3
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s9, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s17, 16
; SI-NEXT: s_and_b32 s7, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB45_4
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB45_5
-; SI-NEXT: .LBB45_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB45_2
-; SI-NEXT: .LBB45_4:
+; SI-NEXT: s_branch .LBB45_6
+; SI-NEXT: .LBB45_5:
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: .LBB45_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -5121,15 +5596,21 @@ define inreg <4 x bfloat> @bitcast_f64_to_v4bf16_scalar(double inreg %a, i32 inr
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_3
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB45_4
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_3:
-; VI-NEXT: s_branch .LBB45_2
-; VI-NEXT: .LBB45_4:
+; VI-NEXT: .LBB45_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5138,15 +5619,21 @@ define inreg <4 x bfloat> @bitcast_f64_to_v4bf16_scalar(double inreg %a, i32 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB45_4
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_3:
-; GFX9-NEXT: s_branch .LBB45_2
-; GFX9-NEXT: .LBB45_4:
+; GFX9-NEXT: .LBB45_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5156,15 +5643,18 @@ define inreg <4 x bfloat> @bitcast_f64_to_v4bf16_scalar(double inreg %a, i32 inr
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_4
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB45_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[0:1], 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_3:
-; GFX11-NEXT: s_branch .LBB45_2
; GFX11-NEXT: .LBB45_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -5463,14 +5953,23 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB47_4
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_cbranch_execnz .LBB47_3
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -5483,20 +5982,25 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB47_3: ; %end
+; SI-NEXT: .LBB47_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SI-NEXT: s_branch .LBB47_2
;
; VI-LABEL: bitcast_v4bf16_to_f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v4, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v4
@@ -5536,9 +6040,7 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -5547,10 +6049,18 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -5592,9 +6102,7 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -5604,11 +6112,16 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-TRUE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -5650,8 +6163,6 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB47_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB47_2
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -5661,11 +6172,16 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-FAKE16-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB47_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -5711,8 +6227,6 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB47_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB47_2
; GFX11-FAKE16-NEXT: .LBB47_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -5943,7 +6457,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s14, s17, 24
; SI-NEXT: s_lshr_b32 s13, s17, 16
@@ -5951,8 +6465,22 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[10:11], s[16:17], 1.0
; SI-NEXT: v_lshr_b64 v[3:4], v[10:11], 24
; SI-NEXT: v_lshr_b64 v[8:9], v[10:11], 16
@@ -5960,16 +6488,8 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v11
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v11
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v11, s17
; SI-NEXT: v_mov_b32_e32 v10, s16
; SI-NEXT: v_mov_b32_e32 v7, s14
@@ -5978,7 +6498,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v8, s6
; SI-NEXT: v_mov_b32_e32 v3, s4
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v10
; SI-NEXT: v_mov_b32_e32 v2, v8
; SI-NEXT: v_mov_b32_e32 v4, v11
@@ -5988,7 +6508,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s9, s17, 24
@@ -5996,8 +6516,22 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; VI-NEXT: s_lshr_b32 s5, s17, 8
; VI-NEXT: s_lshr_b32 s11, s16, 16
; VI-NEXT: s_lshr_b32 s10, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[8:9], s[16:17], 1.0
; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; VI-NEXT: v_lshrrev_b32_e32 v7, 24, v9
@@ -6005,16 +6539,8 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v8, s16
; VI-NEXT: v_mov_b32_e32 v9, s17
; VI-NEXT: v_mov_b32_e32 v2, s11
@@ -6023,7 +6549,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; VI-NEXT: v_mov_b32_e32 v7, s9
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_mov_b32_e32 v0, v8
; VI-NEXT: v_mov_b32_e32 v4, v9
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6032,7 +6558,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s9, s17, 24
@@ -6040,8 +6566,22 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX9-NEXT: s_lshr_b32 s5, s17, 8
; GFX9-NEXT: s_lshr_b32 s11, s16, 16
; GFX9-NEXT: s_lshr_b32 s10, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[8:9], s[16:17], 1.0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX9-NEXT: v_lshrrev_b32_e32 v7, 24, v9
@@ -6049,16 +6589,8 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s11
@@ -6067,7 +6599,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX9-NEXT: v_mov_b32_e32 v7, s9
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v8
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6077,7 +6609,7 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s6, s1, 24
@@ -6085,9 +6617,22 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX11-NEXT: s_lshr_b32 s3, s1, 8
; GFX11-NEXT: s_lshr_b32 s8, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB49_3
+; GFX11-NEXT: .LBB49_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB49_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[8:9], s[0:1], 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
@@ -6096,22 +6641,14 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB49_5
-; GFX11-NEXT: .LBB49_3:
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB49_2
-; GFX11-NEXT: .LBB49_4:
+; GFX11-NEXT: s_branch .LBB49_6
+; GFX11-NEXT: .LBB49_5:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s6
; GFX11-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-NEXT: .LBB49_5: ; %end
+; GFX11-NEXT: .LBB49_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v9
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -6470,7 +7007,7 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -6490,8 +7027,17 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -6518,19 +7064,16 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: .LBB51_3: ; %end
+; SI-NEXT: .LBB51_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v8i8_to_f64_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -6545,8 +7088,17 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6568,17 +7120,14 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v8i8_to_f64_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -6593,8 +7142,17 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB51_3
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -6615,18 +7173,15 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB51_3: ; %end
+; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB51_2
;
; GFX11-LABEL: bitcast_v8i8_to_f64_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -6640,9 +7195,17 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_3
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB51_3
+; GFX11-NEXT: .LBB51_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB51_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -6667,11 +7230,8 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB51_3: ; %end
+; GFX11-NEXT: .LBB51_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB51_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6768,71 +7328,92 @@ define inreg <2 x float> @bitcast_v2i32_to_v2f32_scalar(<2 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v2i32_to_v2f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_4
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_3
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB53_3: ; %end
+; VI-NEXT: .LBB53_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_4:
-; VI-NEXT: s_branch .LBB53_2
;
; GFX9-LABEL: bitcast_v2i32_to_v2f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_3
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB53_3: ; %end
+; GFX9-NEXT: .LBB53_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_4:
-; GFX9-NEXT: s_branch .LBB53_2
;
; GFX11-LABEL: bitcast_v2i32_to_v2f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB53_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB53_3
-; GFX11-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB53_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB53_3: ; %end
+; GFX11-NEXT: .LBB53_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB53_4:
-; GFX11-NEXT: s_branch .LBB53_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6928,16 +7509,22 @@ define inreg <2 x i32> @bitcast_v2f32_to_v2i32_scalar(<2 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -6946,16 +7533,22 @@ define inreg <2 x i32> @bitcast_v2f32_to_v2i32_scalar(<2 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -6964,16 +7557,22 @@ define inreg <2 x i32> @bitcast_v2f32_to_v2i32_scalar(<2 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -6983,16 +7582,19 @@ define inreg <2 x i32> @bitcast_v2f32_to_v2i32_scalar(<2 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7106,17 +7708,27 @@ define inreg <4 x i16> @bitcast_v2i32_to_v4i16_scalar(<2 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_lshr_b32 s8, s17, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -7126,63 +7738,74 @@ define inreg <4 x i16> @bitcast_v2i32_to_v4i16_scalar(<2 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v2i32_to_v4i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v2i32_to_v4i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_3
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB57_3: ; %end
+; GFX9-NEXT: .LBB57_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: s_branch .LBB57_2
;
; GFX11-LABEL: bitcast_v2i32_to_v4i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_3
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB57_3: ; %end
+; GFX11-NEXT: .LBB57_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_4:
-; GFX11-NEXT: s_branch .LBB57_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7315,7 +7938,7 @@ define inreg <2 x i32> @bitcast_v4i16_to_v2i32_scalar(<4 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -7323,8 +7946,17 @@ define inreg <2 x i32> @bitcast_v4i16_to_v2i32_scalar(<4 x i16> inreg %a, i32 in
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -7335,22 +7967,27 @@ define inreg <2 x i32> @bitcast_v4i16_to_v2i32_scalar(<4 x i16> inreg %a, i32 in
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v4i16_to_v2i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -7361,27 +7998,31 @@ define inreg <2 x i32> @bitcast_v4i16_to_v2i32_scalar(<4 x i16> inreg %a, i32 in
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v4i16_to_v2i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -7391,16 +8032,19 @@ define inreg <2 x i32> @bitcast_v4i16_to_v2i32_scalar(<4 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7514,17 +8158,27 @@ define inreg <4 x half> @bitcast_v2i32_to_v4f16_scalar(<2 x i32> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB61_4
+; SI-NEXT: s_cbranch_scc0 .LBB61_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB61_3
-; SI-NEXT: .LBB61_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB61_3
+; SI-NEXT: .LBB61_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB61_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB61_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
; SI-NEXT: s_lshr_b32 s8, s17, 16
-; SI-NEXT: .LBB61_3: ; %end
+; SI-NEXT: .LBB61_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -7534,63 +8188,74 @@ define inreg <4 x half> @bitcast_v2i32_to_v4f16_scalar(<2 x i32> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB61_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB61_2
;
; VI-LABEL: bitcast_v2i32_to_v4f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB61_4
+; VI-NEXT: s_cbranch_scc0 .LBB61_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB61_3
-; VI-NEXT: .LBB61_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB61_3
+; VI-NEXT: .LBB61_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB61_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB61_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB61_3: ; %end
+; VI-NEXT: .LBB61_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB61_4:
-; VI-NEXT: s_branch .LBB61_2
;
; GFX9-LABEL: bitcast_v2i32_to_v4f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB61_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB61_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB61_3
-; GFX9-NEXT: .LBB61_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB61_3
+; GFX9-NEXT: .LBB61_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB61_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB61_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB61_3: ; %end
+; GFX9-NEXT: .LBB61_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB61_4:
-; GFX9-NEXT: s_branch .LBB61_2
;
; GFX11-LABEL: bitcast_v2i32_to_v4f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB61_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB61_3
-; GFX11-NEXT: .LBB61_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB61_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB61_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB61_3: ; %end
+; GFX11-NEXT: .LBB61_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB61_4:
-; GFX11-NEXT: s_branch .LBB61_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7732,7 +8397,7 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB63_3
+; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -7740,8 +8405,17 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB63_4
-; SI-NEXT: .LBB63_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB63_3
+; SI-NEXT: .LBB63_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB63_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB63_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
@@ -7759,10 +8433,7 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB63_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB63_2
-; SI-NEXT: .LBB63_4:
+; SI-NEXT: .LBB63_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -7771,10 +8442,18 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB63_3
+; VI-NEXT: s_cbranch_scc0 .LBB63_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB63_4
-; VI-NEXT: .LBB63_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB63_3
+; VI-NEXT: .LBB63_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB63_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB63_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -7787,9 +8466,7 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB63_3:
-; VI-NEXT: s_branch .LBB63_2
-; VI-NEXT: .LBB63_4:
+; VI-NEXT: .LBB63_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -7798,17 +8475,23 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB63_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB63_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB63_4
-; GFX9-NEXT: .LBB63_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB63_3
+; GFX9-NEXT: .LBB63_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB63_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB63_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB63_3:
-; GFX9-NEXT: s_branch .LBB63_2
-; GFX9-NEXT: .LBB63_4:
+; GFX9-NEXT: .LBB63_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -7818,16 +8501,19 @@ define inreg <2 x i32> @bitcast_v4f16_to_v2i32_scalar(<4 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB63_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB63_4
-; GFX11-NEXT: .LBB63_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB63_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB63_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB63_3:
-; GFX11-NEXT: s_branch .LBB63_2
; GFX11-NEXT: .LBB63_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -7951,21 +8637,33 @@ define inreg <4 x bfloat> @bitcast_v2i32_to_v4bf16_scalar(<2 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB65_4
+; SI-NEXT: s_cbranch_scc0 .LBB65_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s17, 16
; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB65_3
-; SI-NEXT: .LBB65_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB65_3
+; SI-NEXT: .LBB65_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB65_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB65_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s4, s16, 3
; SI-NEXT: s_add_i32 s5, s17, 3
; SI-NEXT: s_and_b32 s7, s5, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s8, s4, 16
-; SI-NEXT: .LBB65_3: ; %end
+; SI-NEXT: .LBB65_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
@@ -7975,65 +8673,74 @@ define inreg <4 x bfloat> @bitcast_v2i32_to_v4bf16_scalar(<2 x i32> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB65_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB65_2
;
; VI-LABEL: bitcast_v2i32_to_v4bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB65_4
+; VI-NEXT: s_cbranch_scc0 .LBB65_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB65_3
-; VI-NEXT: .LBB65_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB65_3
+; VI-NEXT: .LBB65_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB65_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB65_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB65_3: ; %end
+; VI-NEXT: .LBB65_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB65_4:
-; VI-NEXT: s_branch .LBB65_2
;
; GFX9-LABEL: bitcast_v2i32_to_v4bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB65_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB65_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB65_3
-; GFX9-NEXT: .LBB65_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB65_3
+; GFX9-NEXT: .LBB65_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB65_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB65_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB65_3: ; %end
+; GFX9-NEXT: .LBB65_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB65_4:
-; GFX9-NEXT: s_branch .LBB65_2
;
; GFX11-LABEL: bitcast_v2i32_to_v4bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB65_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB65_3
-; GFX11-NEXT: .LBB65_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB65_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB65_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB65_3: ; %end
+; GFX11-NEXT: .LBB65_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB65_4:
-; GFX11-NEXT: s_branch .LBB65_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8329,14 +9036,23 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB67_4
+; SI-NEXT: s_cbranch_scc0 .LBB67_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_cbranch_execnz .LBB67_3
-; SI-NEXT: .LBB67_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB67_3
+; SI-NEXT: .LBB67_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: .LBB67_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB67_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -8349,20 +9065,25 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB67_3: ; %end
+; SI-NEXT: .LBB67_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB67_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SI-NEXT: s_branch .LBB67_2
;
; VI-LABEL: bitcast_v4bf16_to_v2i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB67_3
+; VI-NEXT: s_cbranch_scc0 .LBB67_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB67_4
-; VI-NEXT: .LBB67_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB67_3
+; VI-NEXT: .LBB67_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB67_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB67_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v4, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v4
@@ -8402,9 +9123,7 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB67_3:
-; VI-NEXT: s_branch .LBB67_2
-; VI-NEXT: .LBB67_4:
+; VI-NEXT: .LBB67_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -8413,10 +9132,18 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB67_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB67_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB67_4
-; GFX9-NEXT: .LBB67_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB67_3
+; GFX9-NEXT: .LBB67_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB67_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB67_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -8458,9 +9185,7 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB67_3:
-; GFX9-NEXT: s_branch .LBB67_2
-; GFX9-NEXT: .LBB67_4:
+; GFX9-NEXT: .LBB67_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -8470,11 +9195,16 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-TRUE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -8516,8 +9246,6 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB67_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB67_2
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -8527,11 +9255,16 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB67_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB67_4
-; GFX11-FAKE16-NEXT: .LBB67_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB67_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB67_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -8577,8 +9310,6 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB67_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB67_2
; GFX11-FAKE16-NEXT: .LBB67_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -8814,7 +9545,7 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB69_4
+; SI-NEXT: s_cbranch_scc0 .LBB69_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s17, 24
; SI-NEXT: s_lshr_b32 s13, s17, 16
@@ -8822,8 +9553,22 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB69_3
-; SI-NEXT: .LBB69_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB69_3
+; SI-NEXT: .LBB69_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB69_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB69_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
@@ -8832,7 +9577,7 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; SI-NEXT: s_lshr_b32 s12, s17, 24
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b32 s14, s17, 8
-; SI-NEXT: .LBB69_3: ; %end
+; SI-NEXT: .LBB69_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -8842,20 +9587,12 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; SI-NEXT: v_mov_b32_e32 v6, s13
; SI-NEXT: v_mov_b32_e32 v7, s12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB69_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB69_2
;
; VI-LABEL: bitcast_v2i32_to_v8i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB69_4
+; VI-NEXT: s_cbranch_scc0 .LBB69_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s5, s17, 24
@@ -8863,8 +9600,22 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB69_3
-; VI-NEXT: .LBB69_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB69_3
+; VI-NEXT: .LBB69_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: .LBB69_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB69_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
@@ -8873,7 +9624,7 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
-; VI-NEXT: .LBB69_3: ; %end
+; VI-NEXT: .LBB69_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s11
; VI-NEXT: v_mov_b32_e32 v2, s10
@@ -8883,20 +9634,12 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v7, s5
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB69_4:
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: s_branch .LBB69_2
;
; GFX9-LABEL: bitcast_v2i32_to_v8i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB69_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
@@ -8904,8 +9647,22 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB69_3
-; GFX9-NEXT: .LBB69_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB69_3
+; GFX9-NEXT: .LBB69_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: .LBB69_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
@@ -8914,7 +9671,7 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: .LBB69_3: ; %end
+; GFX9-NEXT: .LBB69_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s11
; GFX9-NEXT: v_mov_b32_e32 v2, s10
@@ -8924,21 +9681,13 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB69_4:
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: s_branch .LBB69_2
;
; GFX11-LABEL: bitcast_v2i32_to_v8i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
@@ -8946,9 +9695,22 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX11-NEXT: s_lshr_b32 s5, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB69_3
-; GFX11-NEXT: .LBB69_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_lshr_b32 s4, s1, 16
@@ -8957,21 +9719,13 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX11-NEXT: s_lshr_b32 s5, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: .LBB69_3: ; %end
+; GFX11-NEXT: .LBB69_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s7
; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s3
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_4:
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB69_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9327,7 +10081,7 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB71_4
+; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -9347,8 +10101,17 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB71_3
-; SI-NEXT: .LBB71_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB71_3
+; SI-NEXT: .LBB71_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB71_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB71_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -9375,19 +10138,16 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: .LBB71_3: ; %end
+; SI-NEXT: .LBB71_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB71_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB71_2
;
; VI-LABEL: bitcast_v8i8_to_v2i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB71_4
+; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -9402,8 +10162,17 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB71_3
-; VI-NEXT: .LBB71_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB71_3
+; VI-NEXT: .LBB71_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB71_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB71_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9425,17 +10194,14 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB71_3: ; %end
+; VI-NEXT: .LBB71_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB71_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB71_2
;
; GFX9-LABEL: bitcast_v8i8_to_v2i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -9450,8 +10216,17 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB71_3
-; GFX9-NEXT: .LBB71_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB71_3
+; GFX9-NEXT: .LBB71_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB71_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -9472,18 +10247,15 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB71_3: ; %end
+; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB71_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB71_2
;
; GFX11-LABEL: bitcast_v8i8_to_v2i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -9497,9 +10269,17 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB71_3
-; GFX11-NEXT: .LBB71_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -9524,11 +10304,8 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB71_3: ; %end
+; GFX11-NEXT: .LBB71_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB71_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB71_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9638,27 +10415,33 @@ define inreg <4 x i16> @bitcast_v2f32_to_v4i16_scalar(<2 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB73_3
+; SI-NEXT: s_cbranch_scc0 .LBB73_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB73_4
-; SI-NEXT: .LBB73_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB73_3
+; SI-NEXT: .LBB73_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB73_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB73_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: s_branch .LBB73_5
-; SI-NEXT: .LBB73_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB73_2
-; SI-NEXT: .LBB73_4:
+; SI-NEXT: s_branch .LBB73_6
+; SI-NEXT: .LBB73_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: .LBB73_5: ; %end
+; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -9671,16 +10454,22 @@ define inreg <4 x i16> @bitcast_v2f32_to_v4i16_scalar(<2 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB73_3
+; VI-NEXT: s_cbranch_scc0 .LBB73_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB73_4
-; VI-NEXT: .LBB73_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB73_3
+; VI-NEXT: .LBB73_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB73_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB73_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB73_3:
-; VI-NEXT: s_branch .LBB73_2
-; VI-NEXT: .LBB73_4:
+; VI-NEXT: .LBB73_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -9689,16 +10478,22 @@ define inreg <4 x i16> @bitcast_v2f32_to_v4i16_scalar(<2 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB73_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB73_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB73_4
-; GFX9-NEXT: .LBB73_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB73_3
+; GFX9-NEXT: .LBB73_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB73_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB73_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB73_3:
-; GFX9-NEXT: s_branch .LBB73_2
-; GFX9-NEXT: .LBB73_4:
+; GFX9-NEXT: .LBB73_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -9708,16 +10503,19 @@ define inreg <4 x i16> @bitcast_v2f32_to_v4i16_scalar(<2 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB73_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB73_4
-; GFX11-NEXT: .LBB73_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB73_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB73_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB73_3:
-; GFX11-NEXT: s_branch .LBB73_2
; GFX11-NEXT: .LBB73_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -9853,7 +10651,7 @@ define inreg <2 x float> @bitcast_v4i16_to_v2f32_scalar(<4 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB75_4
+; SI-NEXT: s_cbranch_scc0 .LBB75_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -9861,8 +10659,17 @@ define inreg <2 x float> @bitcast_v4i16_to_v2f32_scalar(<4 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB75_3
-; SI-NEXT: .LBB75_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB75_3
+; SI-NEXT: .LBB75_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB75_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB75_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -9873,22 +10680,27 @@ define inreg <2 x float> @bitcast_v4i16_to_v2f32_scalar(<4 x i16> inreg %a, i32
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: .LBB75_3: ; %end
+; SI-NEXT: .LBB75_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB75_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB75_2
;
; VI-LABEL: bitcast_v4i16_to_v2f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB75_4
+; VI-NEXT: s_cbranch_scc0 .LBB75_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB75_3
-; VI-NEXT: .LBB75_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB75_3
+; VI-NEXT: .LBB75_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB75_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB75_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -9899,27 +10711,31 @@ define inreg <2 x float> @bitcast_v4i16_to_v2f32_scalar(<4 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB75_3: ; %end
+; VI-NEXT: .LBB75_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB75_4:
-; VI-NEXT: s_branch .LBB75_2
;
; GFX9-LABEL: bitcast_v4i16_to_v2f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB75_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB75_4
-; GFX9-NEXT: .LBB75_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB75_3
+; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB75_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB75_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB75_3:
-; GFX9-NEXT: s_branch .LBB75_2
-; GFX9-NEXT: .LBB75_4:
+; GFX9-NEXT: .LBB75_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -9929,16 +10745,19 @@ define inreg <2 x float> @bitcast_v4i16_to_v2f32_scalar(<4 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB75_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB75_4
-; GFX11-NEXT: .LBB75_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB75_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB75_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB75_3:
-; GFX11-NEXT: s_branch .LBB75_2
; GFX11-NEXT: .LBB75_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -10051,27 +10870,33 @@ define inreg <4 x half> @bitcast_v2f32_to_v4f16_scalar(<2 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB77_3
+; SI-NEXT: s_cbranch_scc0 .LBB77_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB77_4
-; SI-NEXT: .LBB77_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB77_3
+; SI-NEXT: .LBB77_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB77_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB77_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: s_branch .LBB77_5
-; SI-NEXT: .LBB77_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB77_2
-; SI-NEXT: .LBB77_4:
+; SI-NEXT: s_branch .LBB77_6
+; SI-NEXT: .LBB77_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v3, s8
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: .LBB77_5: ; %end
+; SI-NEXT: .LBB77_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -10084,16 +10909,22 @@ define inreg <4 x half> @bitcast_v2f32_to_v4f16_scalar(<2 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB77_3
+; VI-NEXT: s_cbranch_scc0 .LBB77_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB77_4
-; VI-NEXT: .LBB77_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB77_3
+; VI-NEXT: .LBB77_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB77_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB77_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB77_3:
-; VI-NEXT: s_branch .LBB77_2
-; VI-NEXT: .LBB77_4:
+; VI-NEXT: .LBB77_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -10102,16 +10933,22 @@ define inreg <4 x half> @bitcast_v2f32_to_v4f16_scalar(<2 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB77_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB77_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB77_4
-; GFX9-NEXT: .LBB77_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB77_3
+; GFX9-NEXT: .LBB77_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB77_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB77_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB77_3:
-; GFX9-NEXT: s_branch .LBB77_2
-; GFX9-NEXT: .LBB77_4:
+; GFX9-NEXT: .LBB77_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -10121,16 +10958,19 @@ define inreg <4 x half> @bitcast_v2f32_to_v4f16_scalar(<2 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB77_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB77_4
-; GFX11-NEXT: .LBB77_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB77_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB77_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB77_3:
-; GFX11-NEXT: s_branch .LBB77_2
; GFX11-NEXT: .LBB77_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -10275,7 +11115,7 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s8, s17, 16
; SI-NEXT: s_lshr_b32 s9, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB79_3
+; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s9, 16
@@ -10283,8 +11123,17 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s8, 16
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB79_4
-; SI-NEXT: .LBB79_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB79_3
+; SI-NEXT: .LBB79_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB79_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB79_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s9
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s8
@@ -10302,10 +11151,7 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB79_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB79_2
-; SI-NEXT: .LBB79_4:
+; SI-NEXT: .LBB79_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -10314,10 +11160,18 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB79_3
+; VI-NEXT: s_cbranch_scc0 .LBB79_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB79_4
-; VI-NEXT: .LBB79_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB79_3
+; VI-NEXT: .LBB79_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB79_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB79_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -10330,9 +11184,7 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB79_3:
-; VI-NEXT: s_branch .LBB79_2
-; VI-NEXT: .LBB79_4:
+; VI-NEXT: .LBB79_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -10341,17 +11193,23 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB79_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB79_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB79_4
-; GFX9-NEXT: .LBB79_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB79_3
+; GFX9-NEXT: .LBB79_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB79_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB79_3:
-; GFX9-NEXT: s_branch .LBB79_2
-; GFX9-NEXT: .LBB79_4:
+; GFX9-NEXT: .LBB79_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -10361,16 +11219,19 @@ define inreg <2 x float> @bitcast_v4f16_to_v2f32_scalar(<4 x half> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB79_4
-; GFX11-NEXT: .LBB79_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB79_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB79_3:
-; GFX11-NEXT: s_branch .LBB79_2
; GFX11-NEXT: .LBB79_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -10493,33 +11354,39 @@ define inreg <4 x bfloat> @bitcast_v2f32_to_v4bf16_scalar(<2 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB81_3
+; SI-NEXT: s_cbranch_scc0 .LBB81_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s17, 16
; SI-NEXT: s_and_b32 s8, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB81_4
-; SI-NEXT: .LBB81_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB81_3
+; SI-NEXT: .LBB81_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB81_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB81_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB81_5
-; SI-NEXT: .LBB81_3:
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB81_2
-; SI-NEXT: .LBB81_4:
+; SI-NEXT: s_branch .LBB81_6
+; SI-NEXT: .LBB81_5:
; SI-NEXT: v_mov_b32_e32 v0, s9
; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: v_mov_b32_e32 v2, s7
; SI-NEXT: v_mov_b32_e32 v3, s6
-; SI-NEXT: .LBB81_5: ; %end
+; SI-NEXT: .LBB81_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -10534,16 +11401,22 @@ define inreg <4 x bfloat> @bitcast_v2f32_to_v4bf16_scalar(<2 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB81_3
+; VI-NEXT: s_cbranch_scc0 .LBB81_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB81_4
-; VI-NEXT: .LBB81_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB81_3
+; VI-NEXT: .LBB81_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB81_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB81_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB81_3:
-; VI-NEXT: s_branch .LBB81_2
-; VI-NEXT: .LBB81_4:
+; VI-NEXT: .LBB81_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -10552,16 +11425,22 @@ define inreg <4 x bfloat> @bitcast_v2f32_to_v4bf16_scalar(<2 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB81_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB81_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB81_4
-; GFX9-NEXT: .LBB81_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB81_3
+; GFX9-NEXT: .LBB81_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB81_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB81_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB81_3:
-; GFX9-NEXT: s_branch .LBB81_2
-; GFX9-NEXT: .LBB81_4:
+; GFX9-NEXT: .LBB81_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -10571,16 +11450,19 @@ define inreg <4 x bfloat> @bitcast_v2f32_to_v4bf16_scalar(<2 x float> inreg %a,
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB81_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB81_4
-; GFX11-NEXT: .LBB81_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB81_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB81_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB81_3:
-; GFX11-NEXT: s_branch .LBB81_2
; GFX11-NEXT: .LBB81_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -10879,14 +11761,23 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB83_4
+; SI-NEXT: s_cbranch_scc0 .LBB83_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v8
; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
-; SI-NEXT: s_cbranch_execnz .LBB83_3
-; SI-NEXT: .LBB83_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB83_3
+; SI-NEXT: .LBB83_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SI-NEXT: .LBB83_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB83_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -10899,20 +11790,25 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: .LBB83_3: ; %end
+; SI-NEXT: .LBB83_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB83_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SI-NEXT: s_branch .LBB83_2
;
; VI-LABEL: bitcast_v4bf16_to_v2f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB83_3
+; VI-NEXT: s_cbranch_scc0 .LBB83_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB83_4
-; VI-NEXT: .LBB83_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB83_3
+; VI-NEXT: .LBB83_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB83_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB83_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v4, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v4
@@ -10952,9 +11848,7 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB83_3:
-; VI-NEXT: s_branch .LBB83_2
-; VI-NEXT: .LBB83_4:
+; VI-NEXT: .LBB83_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -10963,10 +11857,18 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB83_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB83_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB83_4
-; GFX9-NEXT: .LBB83_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB83_3
+; GFX9-NEXT: .LBB83_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB83_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -11008,9 +11910,7 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB83_3:
-; GFX9-NEXT: s_branch .LBB83_2
-; GFX9-NEXT: .LBB83_4:
+; GFX9-NEXT: .LBB83_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -11020,11 +11920,16 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-TRUE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -11066,8 +11971,6 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB83_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB83_2
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -11077,11 +11980,16 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB83_4
-; GFX11-FAKE16-NEXT: .LBB83_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB83_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -11127,8 +12035,6 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB83_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB83_2
; GFX11-FAKE16-NEXT: .LBB83_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -11362,7 +12268,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB85_3
+; SI-NEXT: s_cbranch_scc0 .LBB85_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s12, s17, 24
; SI-NEXT: s_lshr_b32 s13, s17, 16
@@ -11370,8 +12276,22 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB85_4
-; SI-NEXT: .LBB85_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[10:11], 0
+; SI-NEXT: s_branch .LBB85_3
+; SI-NEXT: .LBB85_2:
+; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB85_3: ; %Flow
+; SI-NEXT: s_and_b64 s[10:11], s[10:11], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB85_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v11, s17, 1.0
; SI-NEXT: v_add_f32_e64 v10, s16, 1.0
; SI-NEXT: v_lshr_b64 v[3:4], v[10:11], 24
@@ -11380,16 +12300,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v11
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v11
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; SI-NEXT: s_branch .LBB85_5
-; SI-NEXT: .LBB85_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB85_2
-; SI-NEXT: .LBB85_4:
+; SI-NEXT: s_branch .LBB85_6
+; SI-NEXT: .LBB85_5:
; SI-NEXT: v_mov_b32_e32 v10, s16
; SI-NEXT: v_mov_b32_e32 v11, s17
; SI-NEXT: v_mov_b32_e32 v5, s14
@@ -11398,7 +12310,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v8, s6
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: .LBB85_5: ; %end
+; SI-NEXT: .LBB85_6: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v10
; SI-NEXT: v_mov_b32_e32 v2, v8
; SI-NEXT: v_mov_b32_e32 v4, v11
@@ -11408,7 +12320,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB85_3
+; VI-NEXT: s_cbranch_scc0 .LBB85_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s5, s17, 24
@@ -11416,8 +12328,22 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s10, s17, 8
; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB85_4
-; VI-NEXT: .LBB85_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB85_3
+; VI-NEXT: .LBB85_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: .LBB85_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB85_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v9, s17, 1.0
; VI-NEXT: v_add_f32_e64 v8, s16, 1.0
; VI-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
@@ -11426,16 +12352,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; VI-NEXT: s_branch .LBB85_5
-; VI-NEXT: .LBB85_3:
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: s_branch .LBB85_2
-; VI-NEXT: .LBB85_4:
+; VI-NEXT: s_branch .LBB85_6
+; VI-NEXT: .LBB85_5:
; VI-NEXT: v_mov_b32_e32 v8, s16
; VI-NEXT: v_mov_b32_e32 v9, s17
; VI-NEXT: v_mov_b32_e32 v1, s11
@@ -11444,7 +12362,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v7, s5
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB85_5: ; %end
+; VI-NEXT: .LBB85_6: ; %end
; VI-NEXT: v_mov_b32_e32 v0, v8
; VI-NEXT: v_mov_b32_e32 v4, v9
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -11453,7 +12371,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB85_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
@@ -11461,8 +12379,22 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB85_4
-; GFX9-NEXT: .LBB85_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB85_3
+; GFX9-NEXT: .LBB85_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: .LBB85_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v9, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v8, s16, 1.0
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
@@ -11471,16 +12403,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX9-NEXT: s_branch .LBB85_5
-; GFX9-NEXT: .LBB85_3:
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: s_branch .LBB85_2
-; GFX9-NEXT: .LBB85_4:
+; GFX9-NEXT: s_branch .LBB85_6
+; GFX9-NEXT: .LBB85_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
@@ -11489,7 +12413,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB85_5: ; %end
+; GFX9-NEXT: .LBB85_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v8
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -11499,7 +12423,7 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB85_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
@@ -11507,9 +12431,22 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
-; GFX11-NEXT: .LBB85_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB85_3
+; GFX11-NEXT: .LBB85_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: .LBB85_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v9, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v8, s0, 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -11519,22 +12456,14 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB85_5
-; GFX11-NEXT: .LBB85_3:
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB85_2
-; GFX11-NEXT: .LBB85_4:
+; GFX11-NEXT: s_branch .LBB85_6
+; GFX11-NEXT: .LBB85_5:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB85_5: ; %end
+; GFX11-NEXT: .LBB85_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v9
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -11893,7 +12822,7 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB87_4
+; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -11913,8 +12842,17 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_cbranch_execnz .LBB87_3
-; SI-NEXT: .LBB87_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB87_3
+; SI-NEXT: .LBB87_2:
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: .LBB87_3: ; %Flow
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 1
+; SI-NEXT: s_cbranch_scc1 .LBB87_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -11941,19 +12879,16 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
-; SI-NEXT: .LBB87_3: ; %end
+; SI-NEXT: .LBB87_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB87_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
-; SI-NEXT: s_branch .LBB87_2
;
; VI-LABEL: bitcast_v8i8_to_v2f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB87_4
+; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -11968,8 +12903,17 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB87_3
-; VI-NEXT: .LBB87_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB87_3
+; VI-NEXT: .LBB87_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB87_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB87_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -11991,17 +12935,14 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB87_3: ; %end
+; VI-NEXT: .LBB87_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB87_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB87_2
;
; GFX9-LABEL: bitcast_v8i8_to_v2f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -12016,8 +12957,17 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB87_3
-; GFX9-NEXT: .LBB87_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB87_3
+; GFX9-NEXT: .LBB87_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB87_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -12038,18 +12988,15 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB87_3: ; %end
+; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB87_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB87_2
;
; GFX11-LABEL: bitcast_v8i8_to_v2f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -12063,9 +13010,17 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB87_3
-; GFX11-NEXT: .LBB87_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB87_3
+; GFX11-NEXT: .LBB87_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB87_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -12090,11 +13045,8 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB87_3: ; %end
+; GFX11-NEXT: .LBB87_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB87_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB87_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -12231,7 +13183,7 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB89_4
+; SI-NEXT: s_cbranch_scc0 .LBB89_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s6, s10, 16
@@ -12240,9 +13192,19 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; SI-NEXT: s_or_b32 s13, s5, s6
; SI-NEXT: s_or_b32 s4, s4, s12
; SI-NEXT: s_lshr_b64 s[6:7], s[12:13], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s13
-; SI-NEXT: s_cbranch_execnz .LBB89_3
-; SI-NEXT: .LBB89_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB89_3
+; SI-NEXT: .LBB89_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB89_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB89_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -12255,7 +13217,7 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s10, s5, 16
-; SI-NEXT: .LBB89_3: ; %end
+; SI-NEXT: .LBB89_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -12265,19 +13227,23 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB89_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB89_2
;
; VI-LABEL: bitcast_v4i16_to_v4f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB89_4
+; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB89_3
-; VI-NEXT: .LBB89_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB89_3
+; VI-NEXT: .LBB89_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB89_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB89_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
@@ -12288,27 +13254,31 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB89_3: ; %end
+; VI-NEXT: .LBB89_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB89_4:
-; VI-NEXT: s_branch .LBB89_2
;
; GFX9-LABEL: bitcast_v4i16_to_v4f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB89_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB89_4
-; GFX9-NEXT: .LBB89_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB89_3
+; GFX9-NEXT: .LBB89_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB89_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB89_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB89_3:
-; GFX9-NEXT: s_branch .LBB89_2
-; GFX9-NEXT: .LBB89_4:
+; GFX9-NEXT: .LBB89_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -12318,16 +13288,19 @@ define inreg <4 x half> @bitcast_v4i16_to_v4f16_scalar(<4 x i16> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB89_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB89_4
-; GFX11-NEXT: .LBB89_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB89_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB89_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB89_3:
-; GFX11-NEXT: s_branch .LBB89_2
; GFX11-NEXT: .LBB89_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -12459,10 +13432,18 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s7, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB91_3
+; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB91_4
-; SI-NEXT: .LBB91_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB91_3
+; SI-NEXT: .LBB91_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB91_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB91_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
@@ -12480,15 +13461,13 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_or_b32_e32 v0, v5, v0
-; SI-NEXT: s_branch .LBB91_5
-; SI-NEXT: .LBB91_3:
-; SI-NEXT: s_branch .LBB91_2
-; SI-NEXT: .LBB91_4:
+; SI-NEXT: s_branch .LBB91_6
+; SI-NEXT: .LBB91_5:
; SI-NEXT: v_mov_b32_e32 v4, s6
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v2, s7
-; SI-NEXT: .LBB91_5: ; %end
+; SI-NEXT: .LBB91_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -12501,10 +13480,18 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB91_3
+; VI-NEXT: s_cbranch_scc0 .LBB91_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB91_4
-; VI-NEXT: .LBB91_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB91_3
+; VI-NEXT: .LBB91_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB91_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB91_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -12517,9 +13504,7 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; VI-NEXT: v_or_b32_e32 v1, v1, v0
; VI-NEXT: v_or_b32_e32 v0, v2, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB91_3:
-; VI-NEXT: s_branch .LBB91_2
-; VI-NEXT: .LBB91_4:
+; VI-NEXT: .LBB91_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -12528,17 +13513,23 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB91_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB91_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB91_4
-; GFX9-NEXT: .LBB91_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB91_3
+; GFX9-NEXT: .LBB91_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB91_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB91_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB91_3:
-; GFX9-NEXT: s_branch .LBB91_2
-; GFX9-NEXT: .LBB91_4:
+; GFX9-NEXT: .LBB91_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -12548,16 +13539,19 @@ define inreg <4 x i16> @bitcast_v4f16_to_v4i16_scalar(<4 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB91_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB91_4
-; GFX11-NEXT: .LBB91_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB91_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB91_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB91_3:
-; GFX11-NEXT: s_branch .LBB91_2
; GFX11-NEXT: .LBB91_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -12695,14 +13689,26 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b32 s10, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB93_4
+; SI-NEXT: s_cbranch_scc0 .LBB93_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s7, s16, 16
; SI-NEXT: s_lshl_b32 s9, s10, 16
; SI-NEXT: s_lshl_b32 s6, s17, 16
; SI-NEXT: s_lshl_b32 s8, s11, 16
-; SI-NEXT: s_cbranch_execnz .LBB93_3
-; SI-NEXT: .LBB93_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB93_3
+; SI-NEXT: .LBB93_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: .LBB93_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB93_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_and_b32 s4, s17, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -12717,7 +13723,7 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s8, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s4, 16
-; SI-NEXT: .LBB93_3: ; %end
+; SI-NEXT: .LBB93_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
@@ -12727,21 +13733,23 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s6
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB93_4:
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: s_branch .LBB93_2
;
; VI-LABEL: bitcast_v4i16_to_v4bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB93_4
+; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB93_3
-; VI-NEXT: .LBB93_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB93_3
+; VI-NEXT: .LBB93_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB93_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB93_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
@@ -12752,27 +13760,31 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB93_3: ; %end
+; VI-NEXT: .LBB93_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB93_4:
-; VI-NEXT: s_branch .LBB93_2
;
; GFX9-LABEL: bitcast_v4i16_to_v4bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB93_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB93_4
-; GFX9-NEXT: .LBB93_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB93_3
+; GFX9-NEXT: .LBB93_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB93_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB93_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB93_3:
-; GFX9-NEXT: s_branch .LBB93_2
-; GFX9-NEXT: .LBB93_4:
+; GFX9-NEXT: .LBB93_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -12782,16 +13794,19 @@ define inreg <4 x bfloat> @bitcast_v4i16_to_v4bf16_scalar(<4 x i16> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB93_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB93_4
-; GFX11-NEXT: .LBB93_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB93_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB93_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB93_3:
-; GFX11-NEXT: s_branch .LBB93_2
; GFX11-NEXT: .LBB93_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -13090,14 +14105,26 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB95_4
+; SI-NEXT: s_cbranch_scc0 .LBB95_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; SI-NEXT: s_cbranch_execnz .LBB95_3
-; SI-NEXT: .LBB95_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB95_3
+; SI-NEXT: .LBB95_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: .LBB95_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB95_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
@@ -13112,7 +14139,7 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[1:2], v[4:5], 16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[0:1], 16
-; SI-NEXT: .LBB95_3: ; %end
+; SI-NEXT: .LBB95_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -13120,21 +14147,23 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB95_4:
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: s_branch .LBB95_2
;
; VI-LABEL: bitcast_v4bf16_to_v4i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB95_3
+; VI-NEXT: s_cbranch_scc0 .LBB95_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB95_4
-; VI-NEXT: .LBB95_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB95_3
+; VI-NEXT: .LBB95_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB95_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB95_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v3, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v3
@@ -13174,9 +14203,7 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB95_3:
-; VI-NEXT: s_branch .LBB95_2
-; VI-NEXT: .LBB95_4:
+; VI-NEXT: .LBB95_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -13185,10 +14212,18 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB95_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB95_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB95_4
-; GFX9-NEXT: .LBB95_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB95_3
+; GFX9-NEXT: .LBB95_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB95_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB95_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -13228,9 +14263,7 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_or_b32 v0, v3, v4, v0
; GFX9-NEXT: v_and_or_b32 v1, v1, v4, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB95_3:
-; GFX9-NEXT: s_branch .LBB95_2
-; GFX9-NEXT: .LBB95_4:
+; GFX9-NEXT: .LBB95_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -13240,11 +14273,16 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-TRUE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -13281,8 +14319,6 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB95_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB95_2
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -13292,11 +14328,16 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB95_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB95_4
-; GFX11-FAKE16-NEXT: .LBB95_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB95_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB95_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -13336,8 +14377,6 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v0, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB95_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB95_2
; GFX11-FAKE16-NEXT: .LBB95_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -13602,7 +14641,7 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB97_4
+; SI-NEXT: s_cbranch_scc0 .LBB97_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -13615,8 +14654,22 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
; SI-NEXT: s_lshr_b32 s9, s5, 8
; SI-NEXT: s_bfe_u32 s7, s14, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB97_3
-; SI-NEXT: .LBB97_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB97_3
+; SI-NEXT: .LBB97_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB97_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB97_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -13633,7 +14686,7 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; SI-NEXT: s_lshr_b32 s7, s5, 24
; SI-NEXT: s_lshr_b32 s14, s5, 16
; SI-NEXT: s_lshr_b32 s9, s5, 8
-; SI-NEXT: .LBB97_3: ; %end
+; SI-NEXT: .LBB97_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v2, s8
@@ -13643,20 +14696,12 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; SI-NEXT: v_mov_b32_e32 v6, s14
; SI-NEXT: v_mov_b32_e32 v7, s7
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB97_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB97_2
;
; VI-LABEL: bitcast_v4i16_to_v8i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB97_4
+; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s10, s17, 24
@@ -13664,9 +14709,24 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; VI-NEXT: s_lshr_b32 s5, s17, 8
; VI-NEXT: s_lshr_b32 s11, s16, 16
; VI-NEXT: s_lshr_b32 s12, s16, 8
+; VI-NEXT: s_mov_b64 s[6:7], 0
; VI-NEXT: s_mov_b32 s9, s17
-; VI-NEXT: s_cbranch_execnz .LBB97_3
-; VI-NEXT: .LBB97_2: ; %cmp.true
+; VI-NEXT: s_branch .LBB97_3
+; VI-NEXT: .LBB97_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: .LBB97_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB97_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s5, s17, 16
; VI-NEXT: s_add_i32 s9, s17, 3
; VI-NEXT: s_add_i32 s8, s5, 3
@@ -13683,7 +14743,7 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; VI-NEXT: s_lshr_b32 s11, s6, 16
; VI-NEXT: s_lshr_b32 s12, s6, 8
; VI-NEXT: s_bfe_u32 s10, s8, 0x80008
-; VI-NEXT: .LBB97_3: ; %end
+; VI-NEXT: .LBB97_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s12
; VI-NEXT: v_mov_b32_e32 v2, s11
@@ -13693,21 +14753,12 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v7, s10
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB97_4:
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: s_branch .LBB97_2
;
; GFX9-LABEL: bitcast_v4i16_to_v8i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
@@ -13715,8 +14766,22 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB97_4
-; GFX9-NEXT: .LBB97_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB97_3
+; GFX9-NEXT: .LBB97_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: .LBB97_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v9, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v8, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
@@ -13725,16 +14790,8 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX9-NEXT: s_branch .LBB97_5
-; GFX9-NEXT: .LBB97_3:
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: s_branch .LBB97_2
-; GFX9-NEXT: .LBB97_4:
+; GFX9-NEXT: s_branch .LBB97_6
+; GFX9-NEXT: .LBB97_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
@@ -13743,7 +14800,7 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB97_5: ; %end
+; GFX9-NEXT: .LBB97_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v8
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -13753,7 +14810,7 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB97_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
@@ -13761,9 +14818,22 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB97_4
-; GFX11-NEXT: .LBB97_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB97_3
+; GFX11-NEXT: .LBB97_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: .LBB97_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB97_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v9, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v8, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -13773,22 +14843,14 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB97_5
-; GFX11-NEXT: .LBB97_3:
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB97_2
-; GFX11-NEXT: .LBB97_4:
+; GFX11-NEXT: s_branch .LBB97_6
+; GFX11-NEXT: .LBB97_5:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB97_5: ; %end
+; GFX11-NEXT: .LBB97_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v9
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -14155,7 +15217,7 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB99_4
+; SI-NEXT: s_cbranch_scc0 .LBB99_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -14177,9 +15239,20 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 16
; SI-NEXT: s_or_b32 s4, s4, s10
; SI-NEXT: s_lshr_b32 s7, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s11
-; SI-NEXT: s_cbranch_execnz .LBB99_3
-; SI-NEXT: .LBB99_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB99_3
+; SI-NEXT: .LBB99_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB99_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB99_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -14208,7 +15281,7 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: .LBB99_3: ; %end
+; SI-NEXT: .LBB99_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -14218,17 +15291,12 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB99_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB99_2
;
; VI-LABEL: bitcast_v8i8_to_v4i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB99_4
+; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -14243,8 +15311,17 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB99_3
-; VI-NEXT: .LBB99_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB99_3
+; VI-NEXT: .LBB99_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB99_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB99_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -14266,17 +15343,14 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB99_3: ; %end
+; VI-NEXT: .LBB99_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB99_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB99_2
;
; GFX9-LABEL: bitcast_v8i8_to_v4i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -14291,8 +15365,17 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB99_3
-; GFX9-NEXT: .LBB99_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB99_3
+; GFX9-NEXT: .LBB99_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB99_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -14313,18 +15396,15 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB99_3: ; %end
+; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB99_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB99_2
;
; GFX11-LABEL: bitcast_v8i8_to_v4i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB99_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -14338,9 +15418,17 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB99_3
-; GFX11-NEXT: .LBB99_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB99_3
+; GFX11-NEXT: .LBB99_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB99_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -14365,11 +15453,8 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB99_3: ; %end
+; GFX11-NEXT: .LBB99_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB99_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB99_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14513,14 +15598,26 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s7, s17, 16
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB101_3
+; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s8, s16, 16
; SI-NEXT: s_lshl_b32 s9, s6, 16
; SI-NEXT: s_lshl_b32 s10, s17, 16
; SI-NEXT: s_lshl_b32 s11, s7, 16
-; SI-NEXT: s_cbranch_execnz .LBB101_4
-; SI-NEXT: .LBB101_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB101_3
+; SI-NEXT: .LBB101_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB101_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB101_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s7
; SI-NEXT: v_cvt_f32_f16_e32 v1, s17
; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
@@ -14537,19 +15634,13 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
-; SI-NEXT: s_branch .LBB101_5
-; SI-NEXT: .LBB101_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB101_2
-; SI-NEXT: .LBB101_4:
+; SI-NEXT: s_branch .LBB101_6
+; SI-NEXT: .LBB101_5:
; SI-NEXT: v_mov_b32_e32 v3, s11
; SI-NEXT: v_mov_b32_e32 v2, s10
; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: .LBB101_5: ; %end
+; SI-NEXT: .LBB101_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -14564,10 +15655,18 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB101_3
+; VI-NEXT: s_cbranch_scc0 .LBB101_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB101_4
-; VI-NEXT: .LBB101_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB101_3
+; VI-NEXT: .LBB101_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB101_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB101_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -14580,9 +15679,7 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v0
; VI-NEXT: v_or_b32_e32 v0, v2, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB101_3:
-; VI-NEXT: s_branch .LBB101_2
-; VI-NEXT: .LBB101_4:
+; VI-NEXT: .LBB101_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -14591,17 +15688,23 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB101_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB101_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB101_4
-; GFX9-NEXT: .LBB101_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB101_3
+; GFX9-NEXT: .LBB101_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB101_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB101_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB101_3:
-; GFX9-NEXT: s_branch .LBB101_2
-; GFX9-NEXT: .LBB101_4:
+; GFX9-NEXT: .LBB101_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -14611,16 +15714,19 @@ define inreg <4 x bfloat> @bitcast_v4f16_to_v4bf16_scalar(<4 x half> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s2, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB101_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-NEXT: s_cbranch_vccnz .LBB101_4
-; GFX11-NEXT: .LBB101_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: .LBB101_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB101_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB101_3:
-; GFX11-NEXT: s_branch .LBB101_2
; GFX11-NEXT: .LBB101_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -14930,7 +16036,7 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; SI-NEXT: s_cbranch_scc0 .LBB103_4
+; SI-NEXT: s_cbranch_scc0 .LBB103_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v11
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
@@ -14938,8 +16044,19 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v12
; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[5:6], v[7:8], 16
-; SI-NEXT: s_cbranch_execnz .LBB103_3
-; SI-NEXT: .LBB103_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB103_3
+; SI-NEXT: .LBB103_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: .LBB103_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB103_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v1
@@ -14954,7 +16071,7 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; SI-NEXT: v_lshr_b64 v[1:2], v[3:4], 16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v7
; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: .LBB103_3: ; %end
+; SI-NEXT: .LBB103_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -14962,20 +16079,23 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB103_4:
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: s_branch .LBB103_2
;
; VI-LABEL: bitcast_v4bf16_to_v4f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB103_3
+; VI-NEXT: s_cbranch_scc0 .LBB103_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB103_4
-; VI-NEXT: .LBB103_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB103_3
+; VI-NEXT: .LBB103_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB103_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB103_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v3, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v3
@@ -15015,9 +16135,7 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB103_3:
-; VI-NEXT: s_branch .LBB103_2
-; VI-NEXT: .LBB103_4:
+; VI-NEXT: .LBB103_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -15026,10 +16144,18 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB103_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB103_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB103_4
-; GFX9-NEXT: .LBB103_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB103_3
+; GFX9-NEXT: .LBB103_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB103_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB103_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -15071,9 +16197,7 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB103_3:
-; GFX9-NEXT: s_branch .LBB103_2
-; GFX9-NEXT: .LBB103_4:
+; GFX9-NEXT: .LBB103_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -15083,11 +16207,16 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-TRUE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -15131,8 +16260,6 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB103_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB103_2
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15142,11 +16269,16 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB103_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB103_4
-; GFX11-FAKE16-NEXT: .LBB103_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: .LBB103_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB103_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -15191,8 +16323,6 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB103_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB103_2
; GFX11-FAKE16-NEXT: .LBB103_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -15461,7 +16591,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; SI-NEXT: s_lshr_b32 s14, s17, 16
; SI-NEXT: s_lshr_b32 s15, s16, 16
; SI-NEXT: s_cmp_lg_u32 s18, 0
-; SI-NEXT: s_cbranch_scc0 .LBB105_3
+; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s15, 16
@@ -15474,8 +16604,22 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 8
; SI-NEXT: s_lshr_b32 s7, s5, 8
; SI-NEXT: s_bfe_u32 s9, s14, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB105_4
-; SI-NEXT: .LBB105_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB105_3
+; SI-NEXT: .LBB105_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB105_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s11, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s11, 1
+; SI-NEXT: s_cbranch_scc1 .LBB105_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s15
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s14
@@ -15497,16 +16641,8 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; SI-NEXT: v_lshr_b64 v[1:2], v[9:10], 8
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v10
; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
-; SI-NEXT: s_branch .LBB105_5
-; SI-NEXT: .LBB105_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB105_2
-; SI-NEXT: .LBB105_4:
+; SI-NEXT: s_branch .LBB105_6
+; SI-NEXT: .LBB105_5:
; SI-NEXT: v_mov_b32_e32 v6, s14
; SI-NEXT: v_mov_b32_e32 v7, s9
; SI-NEXT: v_mov_b32_e32 v10, s5
@@ -15515,7 +16651,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; SI-NEXT: v_mov_b32_e32 v3, s6
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v1, s10
-; SI-NEXT: .LBB105_5: ; %end
+; SI-NEXT: .LBB105_6: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v9
; SI-NEXT: v_mov_b32_e32 v2, v4
; SI-NEXT: v_mov_b32_e32 v4, v10
@@ -15525,7 +16661,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB105_3
+; VI-NEXT: s_cbranch_scc0 .LBB105_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s8, s17, 24
@@ -15533,8 +16669,22 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; VI-NEXT: s_lshr_b32 s5, s17, 8
; VI-NEXT: s_lshr_b32 s11, s16, 16
; VI-NEXT: s_lshr_b32 s9, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB105_4
-; VI-NEXT: .LBB105_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB105_3
+; VI-NEXT: .LBB105_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: .LBB105_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB105_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_add_f16_e32 v6, s4, v0
@@ -15552,15 +16702,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; VI-NEXT: v_bfe_u32 v7, v6, 8, 8
; VI-NEXT: v_mov_b32_e32 v4, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB105_3:
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: s_branch .LBB105_2
-; VI-NEXT: .LBB105_4:
+; VI-NEXT: .LBB105_5:
; VI-NEXT: v_mov_b32_e32 v2, s11
; VI-NEXT: v_mov_b32_e32 v6, s10
; VI-NEXT: v_mov_b32_e32 v0, s16
@@ -15575,7 +16717,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB105_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
@@ -15583,8 +16725,22 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB105_4
-; GFX9-NEXT: .LBB105_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB105_3
+; GFX9-NEXT: .LBB105_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: .LBB105_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v9, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v8, s16, v0 op_sel_hi:[1,0]
@@ -15594,16 +16750,8 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX9-NEXT: s_branch .LBB105_5
-; GFX9-NEXT: .LBB105_3:
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: s_branch .LBB105_2
-; GFX9-NEXT: .LBB105_4:
+; GFX9-NEXT: s_branch .LBB105_6
+; GFX9-NEXT: .LBB105_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
@@ -15612,7 +16760,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB105_5: ; %end
+; GFX9-NEXT: .LBB105_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v8
; GFX9-NEXT: v_mov_b32_e32 v4, v9
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -15622,7 +16770,7 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB105_3
+; GFX11-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
@@ -15630,9 +16778,22 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB105_4
-; GFX11-NEXT: .LBB105_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB105_3
+; GFX11-NEXT: .LBB105_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: ; implicit-def: $sgpr7
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: .LBB105_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB105_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v9, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -15642,22 +16803,14 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-NEXT: s_branch .LBB105_5
-; GFX11-NEXT: .LBB105_3:
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: ; implicit-def: $sgpr5
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: s_branch .LBB105_2
-; GFX11-NEXT: .LBB105_4:
+; GFX11-NEXT: s_branch .LBB105_6
+; GFX11-NEXT: .LBB105_5:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
-; GFX11-NEXT: .LBB105_5: ; %end
+; GFX11-NEXT: .LBB105_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v8
; GFX11-NEXT: v_mov_b32_e32 v4, v9
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -16024,7 +17177,7 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB107_4
+; SI-NEXT: s_cbranch_scc0 .LBB107_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -16046,9 +17199,20 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; SI-NEXT: s_lshr_b64 s[6:7], s[10:11], 16
; SI-NEXT: s_or_b32 s4, s4, s10
; SI-NEXT: s_lshr_b32 s7, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s11
-; SI-NEXT: s_cbranch_execnz .LBB107_3
-; SI-NEXT: .LBB107_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB107_3
+; SI-NEXT: .LBB107_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB107_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB107_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -16077,7 +17241,7 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: .LBB107_3: ; %end
+; SI-NEXT: .LBB107_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -16087,17 +17251,12 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB107_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB107_2
;
; VI-LABEL: bitcast_v8i8_to_v4f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB107_4
+; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -16112,8 +17271,17 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB107_3
-; VI-NEXT: .LBB107_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB107_3
+; VI-NEXT: .LBB107_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB107_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB107_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -16135,17 +17303,14 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB107_3: ; %end
+; VI-NEXT: .LBB107_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB107_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB107_2
;
; GFX9-LABEL: bitcast_v8i8_to_v4f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -16160,8 +17325,17 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB107_3
-; GFX9-NEXT: .LBB107_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB107_3
+; GFX9-NEXT: .LBB107_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB107_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -16182,18 +17356,15 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB107_3: ; %end
+; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB107_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB107_2
;
; GFX11-LABEL: bitcast_v8i8_to_v4f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB107_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -16207,9 +17378,17 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB107_3
-; GFX11-NEXT: .LBB107_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB107_3
+; GFX11-NEXT: .LBB107_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB107_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -16234,11 +17413,8 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB107_3: ; %end
+; GFX11-NEXT: .LBB107_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB107_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB107_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16653,7 +17829,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB109_4
+; SI-NEXT: s_cbranch_scc0 .LBB109_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
; SI-NEXT: v_lshr_b64 v[11:12], v[0:1], 16
@@ -16664,8 +17840,22 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[11:12], 8
; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v12
-; SI-NEXT: s_cbranch_execnz .LBB109_3
-; SI-NEXT: .LBB109_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB109_3
+; SI-NEXT: .LBB109_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: .LBB109_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB109_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v15
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -16683,26 +17873,18 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; SI-NEXT: v_lshr_b64 v[9:10], v[11:12], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[11:12], 8
; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v12
-; SI-NEXT: .LBB109_3: ; %end
+; SI-NEXT: .LBB109_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v11
; SI-NEXT: v_mov_b32_e32 v2, v9
; SI-NEXT: v_mov_b32_e32 v4, v12
; SI-NEXT: v_mov_b32_e32 v5, v8
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB109_4:
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: s_branch .LBB109_2
;
; VI-LABEL: bitcast_v4bf16_to_v8i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s18, 0
-; VI-NEXT: s_cbranch_scc0 .LBB109_3
+; VI-NEXT: s_cbranch_scc0 .LBB109_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s5, s17, 24
@@ -16710,8 +17892,22 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; VI-NEXT: s_lshr_b32 s8, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s9, s16, 8
-; VI-NEXT: s_cbranch_execnz .LBB109_4
-; VI-NEXT: .LBB109_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_branch .LBB109_3
+; VI-NEXT: .LBB109_2:
+; VI-NEXT: s_mov_b64 s[6:7], -1
+; VI-NEXT: ; implicit-def: $sgpr9
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr8
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr5
+; VI-NEXT: .LBB109_3: ; %Flow
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s6, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 1
+; VI-NEXT: s_cbranch_scc1 .LBB109_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x40c00000
; VI-NEXT: v_add_f32_e32 v1, s4, v0
@@ -16759,15 +17955,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v9
; VI-NEXT: v_mov_b32_e32 v4, v8
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB109_3:
-; VI-NEXT: ; implicit-def: $sgpr9
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr5
-; VI-NEXT: s_branch .LBB109_2
-; VI-NEXT: .LBB109_4:
+; VI-NEXT: .LBB109_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v6, s11
; VI-NEXT: v_mov_b32_e32 v2, s10
@@ -16782,7 +17970,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s18, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB109_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s8, s17, 24
@@ -16790,8 +17978,22 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s5, s16, 8
-; GFX9-NEXT: s_cbranch_execnz .LBB109_4
-; GFX9-NEXT: .LBB109_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: s_branch .LBB109_3
+; GFX9-NEXT: .LBB109_2:
+; GFX9-NEXT: s_mov_b64 s[6:7], -1
+; GFX9-NEXT: ; implicit-def: $sgpr5
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr9
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr8
+; GFX9-NEXT: .LBB109_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s6, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s6, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -16839,15 +18041,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v10
; GFX9-NEXT: v_mov_b32_e32 v4, v8
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB109_3:
-; GFX9-NEXT: ; implicit-def: $sgpr5
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr9
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: s_branch .LBB109_2
-; GFX9-NEXT: .LBB109_4:
+; GFX9-NEXT: .LBB109_5:
; GFX9-NEXT: v_mov_b32_e32 v6, s11
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s10
@@ -16863,7 +18057,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
@@ -16871,9 +18065,22 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-TRUE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB109_3
+; GFX11-TRUE16-NEXT: .LBB109_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -16925,15 +18132,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v10
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB109_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: s_branch .LBB109_2
-; GFX11-TRUE16-NEXT: .LBB109_4:
+; GFX11-TRUE16-NEXT: .LBB109_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s5
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s6
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s3
@@ -16945,7 +18144,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 24
@@ -16953,9 +18152,22 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s1, 8
; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s0, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB109_4
-; GFX11-FAKE16-NEXT: .LBB109_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB109_3
+; GFX11-FAKE16-NEXT: .LBB109_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-FAKE16-NEXT: .LBB109_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB109_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
@@ -17007,15 +18219,7 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v10
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB109_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-FAKE16-NEXT: s_branch .LBB109_2
-; GFX11-FAKE16-NEXT: .LBB109_4:
+; GFX11-FAKE16-NEXT: .LBB109_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s3
@@ -17385,7 +18589,7 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s24, 0
-; SI-NEXT: s_cbranch_scc0 .LBB111_4
+; SI-NEXT: s_cbranch_scc0 .LBB111_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -17403,8 +18607,20 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s23, 24
; SI-NEXT: s_or_b32 s9, s5, s4
-; SI-NEXT: s_cbranch_execnz .LBB111_3
-; SI-NEXT: .LBB111_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB111_3
+; SI-NEXT: .LBB111_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB111_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB111_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_and_b32 s4, s20, 0xff
; SI-NEXT: s_lshl_b32 s5, s21, 8
@@ -17435,7 +18651,7 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; SI-NEXT: s_lshl_b32 s6, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s4, 16
-; SI-NEXT: .LBB111_3: ; %end
+; SI-NEXT: .LBB111_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
@@ -17445,18 +18661,12 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s7
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB111_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB111_2
;
; VI-LABEL: bitcast_v8i8_to_v4bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s24, 0
-; VI-NEXT: s_cbranch_scc0 .LBB111_4
+; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v2, s19
@@ -17471,8 +18681,17 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v1, s22, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_cbranch_execnz .LBB111_3
-; VI-NEXT: .LBB111_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB111_3
+; VI-NEXT: .LBB111_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: .LBB111_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB111_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -17494,17 +18713,14 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; VI-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
-; VI-NEXT: .LBB111_3: ; %end
+; VI-NEXT: .LBB111_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB111_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT: s_branch .LBB111_2
;
; GFX9-LABEL: bitcast_v8i8_to_v4bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s24, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v2, s19
@@ -17519,8 +18735,17 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v1, s22, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX9-NEXT: s_cbranch_execnz .LBB111_3
-; GFX9-NEXT: .LBB111_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB111_3
+; GFX9-NEXT: .LBB111_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT: .LBB111_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0004
@@ -17541,18 +18766,15 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v2, 0x300, v2
; GFX9-NEXT: v_add_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB111_3: ; %end
+; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB111_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_branch .LBB111_2
;
; GFX11-LABEL: bitcast_v8i8_to_v4bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s20, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB111_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
@@ -17566,9 +18788,17 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB111_3
-; GFX11-NEXT: .LBB111_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB111_3
+; GFX11-NEXT: .LBB111_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB111_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -17593,11 +18823,8 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB111_3: ; %end
+; GFX11-NEXT: .LBB111_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB111_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: s_branch .LBB111_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
index df100725d890c..ced89bdabff64 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll
@@ -178,10 +178,18 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -204,7 +212,7 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -228,8 +236,6 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v22i32_to_v22f32_scalar:
; VI: ; %bb.0:
@@ -244,10 +250,18 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -270,7 +284,7 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -294,8 +308,6 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v22i32_to_v22f32_scalar:
; GFX9: ; %bb.0:
@@ -310,10 +322,18 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -336,7 +356,7 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -360,8 +380,6 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v22i32_to_v22f32_scalar:
; GFX11: ; %bb.0:
@@ -373,11 +391,16 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -400,7 +423,7 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -414,8 +437,6 @@ define inreg <22 x float> @bitcast_v22i32_to_v22f32_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -624,10 +645,18 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
; SI-NEXT: v_add_f32_e64 v20, s56, 1.0
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -650,10 +679,8 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -686,7 +713,7 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -747,10 +774,18 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
; VI-NEXT: v_add_f32_e64 v20, s56, 1.0
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -773,10 +808,8 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -809,7 +842,7 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -870,10 +903,18 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX9-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -896,10 +937,8 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -932,7 +971,7 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -994,11 +1033,16 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX11-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -1022,8 +1066,6 @@ define inreg <22 x i32> @bitcast_v22f32_to_v22i32_scalar(<22 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1248,10 +1290,18 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1274,7 +1324,7 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1298,8 +1348,6 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v22i32_to_v11i64_scalar:
; VI: ; %bb.0:
@@ -1314,10 +1362,18 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1340,7 +1396,7 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1364,8 +1420,6 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v22i32_to_v11i64_scalar:
; GFX9: ; %bb.0:
@@ -1380,10 +1434,18 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1406,7 +1468,7 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1430,8 +1492,6 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v22i32_to_v11i64_scalar:
; GFX11: ; %bb.0:
@@ -1443,11 +1503,16 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1470,7 +1535,7 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1484,8 +1549,6 @@ define inreg <11 x i64> @bitcast_v22i32_to_v11i64_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1681,10 +1744,18 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1707,7 +1778,7 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1731,8 +1802,6 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v11i64_to_v22i32_scalar:
; VI: ; %bb.0:
@@ -1747,10 +1816,18 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -1773,7 +1850,7 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1797,8 +1874,6 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v11i64_to_v22i32_scalar:
; GFX9: ; %bb.0:
@@ -1813,10 +1888,18 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -1839,7 +1922,7 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1863,8 +1946,6 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v11i64_to_v22i32_scalar:
; GFX11: ; %bb.0:
@@ -1876,11 +1957,16 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -1903,7 +1989,7 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1917,8 +2003,6 @@ define inreg <22 x i32> @bitcast_v11i64_to_v22i32_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2108,10 +2192,18 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2134,7 +2226,7 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2158,8 +2250,6 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v22i32_to_v11f64_scalar:
; VI: ; %bb.0:
@@ -2174,10 +2264,18 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2200,7 +2298,7 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2224,8 +2322,6 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v22i32_to_v11f64_scalar:
; GFX9: ; %bb.0:
@@ -2240,10 +2336,18 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2266,7 +2370,7 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2290,8 +2394,6 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v22i32_to_v11f64_scalar:
; GFX11: ; %bb.0:
@@ -2303,11 +2405,16 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -2330,7 +2437,7 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2344,8 +2451,6 @@ define inreg <11 x double> @bitcast_v22i32_to_v11f64_scalar(<22 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2521,10 +2626,18 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -2536,10 +2649,8 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2572,7 +2683,7 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2633,10 +2744,18 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -2648,10 +2767,8 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -2684,7 +2801,7 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2745,10 +2862,18 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -2760,10 +2885,8 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -2796,7 +2919,7 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -2858,11 +2981,16 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -2875,8 +3003,6 @@ define inreg <22 x i32> @bitcast_v11f64_to_v22i32_scalar(<11 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -3569,7 +3695,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s77, s7, 16
@@ -3593,8 +3719,38 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3639,7 +3795,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s13, s72, 16
; SI-NEXT: s_and_b32 s15, s16, 0xffff
; SI-NEXT: s_or_b32 s13, s15, s13
@@ -3729,30 +3885,6 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v20, s4
; SI-NEXT: v_mov_b32_e32 v21, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v22i32_to_v44i16_scalar:
; VI: ; %bb.0:
@@ -3767,7 +3899,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -3791,8 +3923,38 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -3837,7 +3999,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -3927,30 +4089,6 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v22i32_to_v44i16_scalar:
; GFX9: ; %bb.0:
@@ -3965,7 +4103,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -3989,8 +4127,38 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -4035,7 +4203,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s73
@@ -4081,30 +4249,6 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v22i32_to_v44i16_scalar:
; GFX11: ; %bb.0:
@@ -4116,7 +4260,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s62, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
; GFX11-NEXT: s_lshr_b32 s9, s5, 16
@@ -4140,9 +4284,38 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s62, -1
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -4187,7 +4360,7 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s60
@@ -4223,30 +4396,6 @@ define inreg <44 x i16> @bitcast_v22i32_to_v44i16_scalar(<22 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5220,26 +5369,24 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s34, 0
-; SI-NEXT: v_writelane_b32 v22, s35, 1
-; SI-NEXT: v_writelane_b32 v22, s36, 2
-; SI-NEXT: v_writelane_b32 v22, s37, 3
-; SI-NEXT: v_writelane_b32 v22, s38, 4
-; SI-NEXT: v_writelane_b32 v22, s39, 5
-; SI-NEXT: v_writelane_b32 v22, s48, 6
-; SI-NEXT: v_writelane_b32 v22, s49, 7
-; SI-NEXT: v_writelane_b32 v22, s50, 8
-; SI-NEXT: v_writelane_b32 v22, s51, 9
-; SI-NEXT: v_writelane_b32 v22, s52, 10
-; SI-NEXT: v_writelane_b32 v22, s53, 11
-; SI-NEXT: v_writelane_b32 v22, s54, 12
-; SI-NEXT: v_writelane_b32 v22, s55, 13
-; SI-NEXT: v_writelane_b32 v22, s64, 14
-; SI-NEXT: v_writelane_b32 v22, s65, 15
-; SI-NEXT: v_writelane_b32 v22, s66, 16
-; SI-NEXT: v_writelane_b32 v22, s67, 17
-; SI-NEXT: v_writelane_b32 v22, s30, 18
-; SI-NEXT: v_writelane_b32 v22, s31, 19
+; SI-NEXT: v_writelane_b32 v22, s36, 0
+; SI-NEXT: v_writelane_b32 v22, s37, 1
+; SI-NEXT: v_writelane_b32 v22, s38, 2
+; SI-NEXT: v_writelane_b32 v22, s39, 3
+; SI-NEXT: v_writelane_b32 v22, s48, 4
+; SI-NEXT: v_writelane_b32 v22, s49, 5
+; SI-NEXT: v_writelane_b32 v22, s50, 6
+; SI-NEXT: v_writelane_b32 v22, s51, 7
+; SI-NEXT: v_writelane_b32 v22, s52, 8
+; SI-NEXT: v_writelane_b32 v22, s53, 9
+; SI-NEXT: v_writelane_b32 v22, s54, 10
+; SI-NEXT: v_writelane_b32 v22, s55, 11
+; SI-NEXT: v_writelane_b32 v22, s64, 12
+; SI-NEXT: v_writelane_b32 v22, s65, 13
+; SI-NEXT: v_writelane_b32 v22, s66, 14
+; SI-NEXT: v_writelane_b32 v22, s67, 15
+; SI-NEXT: v_writelane_b32 v22, s30, 16
+; SI-NEXT: v_writelane_b32 v22, s31, 17
; SI-NEXT: v_readfirstlane_b32 s7, v7
; SI-NEXT: v_readfirstlane_b32 s9, v6
; SI-NEXT: v_readfirstlane_b32 s11, v5
@@ -5258,10 +5405,10 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -5272,18 +5419,18 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s89, s90, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -5340,26 +5487,35 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -5452,8 +5608,8 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v22, 18
+; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v22, 16
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -5476,33 +5632,28 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v19, s55
; SI-NEXT: v_mov_b32_e32 v20, s56
; SI-NEXT: v_mov_b32_e32 v21, s57
-; SI-NEXT: v_readlane_b32 s31, v22, 19
-; SI-NEXT: v_readlane_b32 s67, v22, 17
-; SI-NEXT: v_readlane_b32 s66, v22, 16
-; SI-NEXT: v_readlane_b32 s65, v22, 15
-; SI-NEXT: v_readlane_b32 s64, v22, 14
-; SI-NEXT: v_readlane_b32 s55, v22, 13
-; SI-NEXT: v_readlane_b32 s54, v22, 12
-; SI-NEXT: v_readlane_b32 s53, v22, 11
-; SI-NEXT: v_readlane_b32 s52, v22, 10
-; SI-NEXT: v_readlane_b32 s51, v22, 9
-; SI-NEXT: v_readlane_b32 s50, v22, 8
-; SI-NEXT: v_readlane_b32 s49, v22, 7
-; SI-NEXT: v_readlane_b32 s48, v22, 6
-; SI-NEXT: v_readlane_b32 s39, v22, 5
-; SI-NEXT: v_readlane_b32 s38, v22, 4
-; SI-NEXT: v_readlane_b32 s37, v22, 3
-; SI-NEXT: v_readlane_b32 s36, v22, 2
-; SI-NEXT: v_readlane_b32 s35, v22, 1
-; SI-NEXT: v_readlane_b32 s34, v22, 0
+; SI-NEXT: v_readlane_b32 s31, v22, 17
+; SI-NEXT: v_readlane_b32 s67, v22, 15
+; SI-NEXT: v_readlane_b32 s66, v22, 14
+; SI-NEXT: v_readlane_b32 s65, v22, 13
+; SI-NEXT: v_readlane_b32 s64, v22, 12
+; SI-NEXT: v_readlane_b32 s55, v22, 11
+; SI-NEXT: v_readlane_b32 s54, v22, 10
+; SI-NEXT: v_readlane_b32 s53, v22, 9
+; SI-NEXT: v_readlane_b32 s52, v22, 8
+; SI-NEXT: v_readlane_b32 s51, v22, 7
+; SI-NEXT: v_readlane_b32 s50, v22, 6
+; SI-NEXT: v_readlane_b32 s49, v22, 5
+; SI-NEXT: v_readlane_b32 s48, v22, 4
+; SI-NEXT: v_readlane_b32 s39, v22, 3
+; SI-NEXT: v_readlane_b32 s38, v22, 2
+; SI-NEXT: v_readlane_b32 s37, v22, 1
+; SI-NEXT: v_readlane_b32 s36, v22, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v44i16_to_v22i32_scalar:
; VI: ; %bb.0:
@@ -5530,10 +5681,8 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v22, s67, 17
; VI-NEXT: v_writelane_b32 v22, s68, 18
; VI-NEXT: v_writelane_b32 v22, s69, 19
-; VI-NEXT: v_writelane_b32 v22, s70, 20
-; VI-NEXT: v_writelane_b32 v22, s71, 21
-; VI-NEXT: v_writelane_b32 v22, s30, 22
-; VI-NEXT: v_writelane_b32 v22, s31, 23
+; VI-NEXT: v_writelane_b32 v22, s30, 20
+; VI-NEXT: v_writelane_b32 v22, s31, 21
; VI-NEXT: v_readfirstlane_b32 s7, v7
; VI-NEXT: v_readfirstlane_b32 s9, v6
; VI-NEXT: v_readfirstlane_b32 s11, v5
@@ -5548,14 +5697,14 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s78, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -5566,31 +5715,31 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -5634,46 +5783,55 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -5746,8 +5904,8 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s57, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v22, 22
+; VI-NEXT: .LBB15_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v22, 20
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -5770,9 +5928,7 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v19, s55
; VI-NEXT: v_mov_b32_e32 v20, s56
; VI-NEXT: v_mov_b32_e32 v21, s57
-; VI-NEXT: v_readlane_b32 s31, v22, 23
-; VI-NEXT: v_readlane_b32 s71, v22, 21
-; VI-NEXT: v_readlane_b32 s70, v22, 20
+; VI-NEXT: v_readlane_b32 s31, v22, 21
; VI-NEXT: v_readlane_b32 s69, v22, 19
; VI-NEXT: v_readlane_b32 s68, v22, 18
; VI-NEXT: v_readlane_b32 s67, v22, 17
@@ -5798,9 +5954,6 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v44i16_to_v22i32_scalar:
; GFX9: ; %bb.0:
@@ -5874,10 +6027,18 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -5900,10 +6061,8 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v20, s56, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -5936,7 +6095,7 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -6009,11 +6168,16 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -6037,8 +6201,6 @@ define inreg <22 x i32> @bitcast_v44i16_to_v22i32_scalar(<44 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v20, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v21, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6714,7 +6876,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s77, s7, 16
@@ -6738,8 +6900,38 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -6784,7 +6976,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s13, s72, 16
; SI-NEXT: s_and_b32 s15, s16, 0xffff
; SI-NEXT: s_or_b32 s13, s15, s13
@@ -6874,30 +7066,6 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v20, s4
; SI-NEXT: v_mov_b32_e32 v21, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v22i32_to_v44f16_scalar:
; VI: ; %bb.0:
@@ -6912,7 +7080,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -6936,8 +7104,38 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -6982,7 +7180,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -7072,30 +7270,6 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v22i32_to_v44f16_scalar:
; GFX9: ; %bb.0:
@@ -7110,7 +7284,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -7134,8 +7308,38 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -7180,7 +7384,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s73
@@ -7226,30 +7430,6 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v22i32_to_v44f16_scalar:
; GFX11: ; %bb.0:
@@ -7261,7 +7441,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s62, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
; GFX11-NEXT: s_lshr_b32 s9, s5, 16
@@ -7285,9 +7465,38 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s62, -1
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -7332,7 +7541,7 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s60
@@ -7368,30 +7577,6 @@ define inreg <44 x half> @bitcast_v22i32_to_v44f16_scalar(<22 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8464,26 +8649,24 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v32, s34, 0
-; SI-NEXT: v_writelane_b32 v32, s35, 1
-; SI-NEXT: v_writelane_b32 v32, s36, 2
-; SI-NEXT: v_writelane_b32 v32, s37, 3
-; SI-NEXT: v_writelane_b32 v32, s38, 4
-; SI-NEXT: v_writelane_b32 v32, s39, 5
-; SI-NEXT: v_writelane_b32 v32, s48, 6
-; SI-NEXT: v_writelane_b32 v32, s49, 7
-; SI-NEXT: v_writelane_b32 v32, s50, 8
-; SI-NEXT: v_writelane_b32 v32, s51, 9
-; SI-NEXT: v_writelane_b32 v32, s52, 10
-; SI-NEXT: v_writelane_b32 v32, s53, 11
-; SI-NEXT: v_writelane_b32 v32, s54, 12
-; SI-NEXT: v_writelane_b32 v32, s55, 13
-; SI-NEXT: v_writelane_b32 v32, s64, 14
-; SI-NEXT: v_writelane_b32 v32, s65, 15
-; SI-NEXT: v_writelane_b32 v32, s66, 16
-; SI-NEXT: v_writelane_b32 v32, s67, 17
-; SI-NEXT: v_writelane_b32 v32, s30, 18
-; SI-NEXT: v_writelane_b32 v32, s31, 19
+; SI-NEXT: v_writelane_b32 v32, s36, 0
+; SI-NEXT: v_writelane_b32 v32, s37, 1
+; SI-NEXT: v_writelane_b32 v32, s38, 2
+; SI-NEXT: v_writelane_b32 v32, s39, 3
+; SI-NEXT: v_writelane_b32 v32, s48, 4
+; SI-NEXT: v_writelane_b32 v32, s49, 5
+; SI-NEXT: v_writelane_b32 v32, s50, 6
+; SI-NEXT: v_writelane_b32 v32, s51, 7
+; SI-NEXT: v_writelane_b32 v32, s52, 8
+; SI-NEXT: v_writelane_b32 v32, s53, 9
+; SI-NEXT: v_writelane_b32 v32, s54, 10
+; SI-NEXT: v_writelane_b32 v32, s55, 11
+; SI-NEXT: v_writelane_b32 v32, s64, 12
+; SI-NEXT: v_writelane_b32 v32, s65, 13
+; SI-NEXT: v_writelane_b32 v32, s66, 14
+; SI-NEXT: v_writelane_b32 v32, s67, 15
+; SI-NEXT: v_writelane_b32 v32, s30, 16
+; SI-NEXT: v_writelane_b32 v32, s31, 17
; SI-NEXT: v_readfirstlane_b32 s6, v7
; SI-NEXT: v_readfirstlane_b32 s8, v6
; SI-NEXT: v_readfirstlane_b32 s10, v5
@@ -8502,10 +8685,10 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -8516,18 +8699,18 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s79, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -8584,11 +8767,20 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -8598,14 +8790,14 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_lo
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -8762,11 +8954,8 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v20, v21, v20
; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v22
; SI-NEXT: v_or_b32_e32 v21, v23, v21
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -8799,27 +8988,25 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 18
-; SI-NEXT: v_readlane_b32 s31, v32, 19
-; SI-NEXT: v_readlane_b32 s67, v32, 17
-; SI-NEXT: v_readlane_b32 s66, v32, 16
-; SI-NEXT: v_readlane_b32 s65, v32, 15
-; SI-NEXT: v_readlane_b32 s64, v32, 14
-; SI-NEXT: v_readlane_b32 s55, v32, 13
-; SI-NEXT: v_readlane_b32 s54, v32, 12
-; SI-NEXT: v_readlane_b32 s53, v32, 11
-; SI-NEXT: v_readlane_b32 s52, v32, 10
-; SI-NEXT: v_readlane_b32 s51, v32, 9
-; SI-NEXT: v_readlane_b32 s50, v32, 8
-; SI-NEXT: v_readlane_b32 s49, v32, 7
-; SI-NEXT: v_readlane_b32 s48, v32, 6
-; SI-NEXT: v_readlane_b32 s39, v32, 5
-; SI-NEXT: v_readlane_b32 s38, v32, 4
-; SI-NEXT: v_readlane_b32 s37, v32, 3
-; SI-NEXT: v_readlane_b32 s36, v32, 2
-; SI-NEXT: v_readlane_b32 s35, v32, 1
-; SI-NEXT: v_readlane_b32 s34, v32, 0
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 16
+; SI-NEXT: v_readlane_b32 s31, v32, 17
+; SI-NEXT: v_readlane_b32 s67, v32, 15
+; SI-NEXT: v_readlane_b32 s66, v32, 14
+; SI-NEXT: v_readlane_b32 s65, v32, 13
+; SI-NEXT: v_readlane_b32 s64, v32, 12
+; SI-NEXT: v_readlane_b32 s55, v32, 11
+; SI-NEXT: v_readlane_b32 s54, v32, 10
+; SI-NEXT: v_readlane_b32 s53, v32, 9
+; SI-NEXT: v_readlane_b32 s52, v32, 8
+; SI-NEXT: v_readlane_b32 s51, v32, 7
+; SI-NEXT: v_readlane_b32 s50, v32, 6
+; SI-NEXT: v_readlane_b32 s49, v32, 5
+; SI-NEXT: v_readlane_b32 s48, v32, 4
+; SI-NEXT: v_readlane_b32 s39, v32, 3
+; SI-NEXT: v_readlane_b32 s38, v32, 2
+; SI-NEXT: v_readlane_b32 s37, v32, 1
+; SI-NEXT: v_readlane_b32 s36, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -8852,10 +9039,8 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s67, 17
; VI-NEXT: v_writelane_b32 v32, s68, 18
; VI-NEXT: v_writelane_b32 v32, s69, 19
-; VI-NEXT: v_writelane_b32 v32, s70, 20
-; VI-NEXT: v_writelane_b32 v32, s71, 21
-; VI-NEXT: v_writelane_b32 v32, s30, 22
-; VI-NEXT: v_writelane_b32 v32, s31, 23
+; VI-NEXT: v_writelane_b32 v32, s30, 20
+; VI-NEXT: v_writelane_b32 v32, s31, 21
; VI-NEXT: v_readfirstlane_b32 s6, v7
; VI-NEXT: v_readfirstlane_b32 s8, v6
; VI-NEXT: v_readfirstlane_b32 s10, v5
@@ -8870,14 +9055,14 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s90, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -8888,31 +9073,31 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s91, s89, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -8956,38 +9141,47 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v21, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s71
-; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v0, s69
+; VI-NEXT: v_mov_b32_e32 v2, s68
; VI-NEXT: v_add_f16_sdwa v0, v0, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v21
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v21
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s69
+; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v21
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v3, s34
; VI-NEXT: v_add_f16_sdwa v3, v3, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v21
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s35
+; VI-NEXT: v_mov_b32_e32 v4, s31
; VI-NEXT: v_add_f16_sdwa v4, v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v21
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s34
+; VI-NEXT: v_mov_b32_e32 v5, s30
; VI-NEXT: v_add_f16_sdwa v5, v5, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v21
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s31
+; VI-NEXT: v_mov_b32_e32 v6, vcc_hi
; VI-NEXT: v_add_f16_sdwa v6, v6, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v21
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s30
+; VI-NEXT: v_mov_b32_e32 v7, vcc_lo
; VI-NEXT: v_add_f16_sdwa v7, v7, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v21
; VI-NEXT: v_or_b32_e32 v7, v8, v7
@@ -9047,11 +9241,8 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v22, v22, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s6, v21
; VI-NEXT: v_or_b32_e32 v21, v21, v22
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -9084,11 +9275,9 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 22
-; VI-NEXT: v_readlane_b32 s31, v32, 23
-; VI-NEXT: v_readlane_b32 s71, v32, 21
-; VI-NEXT: v_readlane_b32 s70, v32, 20
+; VI-NEXT: .LBB19_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 20
+; VI-NEXT: v_readlane_b32 s31, v32, 21
; VI-NEXT: v_readlane_b32 s69, v32, 19
; VI-NEXT: v_readlane_b32 s68, v32, 18
; VI-NEXT: v_readlane_b32 s67, v32, 17
@@ -9187,10 +9376,18 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v21, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v21 op_sel_hi:[1,0]
@@ -9214,10 +9411,8 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v19, s55, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v20, s56, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v21, s57, v21 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -9250,7 +9445,7 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -9323,11 +9518,16 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -9351,8 +9551,6 @@ define inreg <22 x i32> @bitcast_v44f16_to_v22i32_scalar(<44 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v20, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v21, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9579,10 +9777,18 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
; SI-NEXT: v_add_f32_e64 v20, s56, 1.0
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -9605,10 +9811,8 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9641,7 +9845,7 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -9702,10 +9906,18 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
; VI-NEXT: v_add_f32_e64 v20, s56, 1.0
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -9728,10 +9940,8 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -9764,7 +9974,7 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -9825,10 +10035,18 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX9-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -9851,10 +10069,8 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -9887,7 +10103,7 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -9949,11 +10165,16 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX11-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -9977,8 +10198,6 @@ define inreg <11 x i64> @bitcast_v22f32_to_v11i64_scalar(<22 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -10209,10 +10428,18 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -10235,7 +10462,7 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -10259,8 +10486,6 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v11i64_to_v22f32_scalar:
; VI: ; %bb.0:
@@ -10275,10 +10500,18 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -10301,7 +10534,7 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -10325,8 +10558,6 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v11i64_to_v22f32_scalar:
; GFX9: ; %bb.0:
@@ -10341,10 +10572,18 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -10367,7 +10606,7 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -10391,8 +10630,6 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v11i64_to_v22f32_scalar:
; GFX11: ; %bb.0:
@@ -10404,11 +10641,16 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -10431,7 +10673,7 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10445,8 +10687,6 @@ define inreg <22 x float> @bitcast_v11i64_to_v22f32_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10655,10 +10895,18 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
; SI-NEXT: v_add_f32_e64 v20, s56, 1.0
; SI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -10681,10 +10929,8 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10717,7 +10963,7 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10778,10 +11024,18 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
; VI-NEXT: v_add_f32_e64 v20, s56, 1.0
; VI-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -10804,10 +11058,8 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -10840,7 +11092,7 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10901,10 +11153,18 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX9-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX9-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -10927,10 +11187,8 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10963,7 +11221,7 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -11025,11 +11283,16 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
; GFX11-NEXT: v_add_f32_e64 v20, s56, 1.0
; GFX11-NEXT: v_add_f32_e64 v19, s55, 1.0
@@ -11053,8 +11316,6 @@ define inreg <11 x double> @bitcast_v22f32_to_v11f64_scalar(<22 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -11265,10 +11526,18 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -11280,10 +11549,8 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11316,7 +11583,7 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11377,10 +11644,18 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -11392,10 +11667,8 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -11428,7 +11701,7 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11489,10 +11762,18 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -11504,10 +11785,8 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -11540,7 +11819,7 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -11602,11 +11881,16 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
@@ -11619,8 +11903,6 @@ define inreg <22 x float> @bitcast_v11f64_to_v22f32_scalar(<11 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -12291,7 +12573,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s94, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
@@ -12315,8 +12597,38 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s5, 1.0
; SI-NEXT: v_add_f32_e64 v20, s4, 1.0
; SI-NEXT: v_add_f32_e64 v19, s7, 1.0
@@ -12361,32 +12673,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -12431,7 +12719,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s60
; SI-NEXT: v_mov_b32_e32 v31, s62
; SI-NEXT: v_mov_b32_e32 v32, s72
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
@@ -12513,7 +12801,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -12537,8 +12825,38 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v21, s6, 1.0
; VI-NEXT: v_add_f32_e64 v20, s7, 1.0
; VI-NEXT: v_add_f32_e64 v19, s8, 1.0
@@ -12583,32 +12901,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12653,7 +12947,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s40
; VI-NEXT: v_mov_b32_e32 v23, s15
; VI-NEXT: v_mov_b32_e32 v22, s14
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; VI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
@@ -12713,7 +13007,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -12737,8 +13031,38 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v21, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v20, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v19, s8, 1.0
@@ -12783,32 +13107,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12853,7 +13153,7 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -12909,11 +13209,11 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s29, 16
@@ -12934,9 +13234,38 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v21, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v20, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s6, 1.0
@@ -12981,32 +13310,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -13028,8 +13333,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s9 :: v_dual_mov_b32 v22, s8
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v51.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v50.l
@@ -13064,11 +13369,11 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s29, 16
@@ -13089,9 +13394,38 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v17, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v18, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v19, s6, 1.0
@@ -13136,32 +13470,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v5, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v3, s17
@@ -13183,8 +13493,8 @@ define inreg <44 x i16> @bitcast_v22f32_to_v44i16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s9 :: v_dual_mov_b32 v22, s8
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v1
@@ -14203,26 +14513,24 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s34, 0
-; SI-NEXT: v_writelane_b32 v22, s35, 1
-; SI-NEXT: v_writelane_b32 v22, s36, 2
-; SI-NEXT: v_writelane_b32 v22, s37, 3
-; SI-NEXT: v_writelane_b32 v22, s38, 4
-; SI-NEXT: v_writelane_b32 v22, s39, 5
-; SI-NEXT: v_writelane_b32 v22, s48, 6
-; SI-NEXT: v_writelane_b32 v22, s49, 7
-; SI-NEXT: v_writelane_b32 v22, s50, 8
-; SI-NEXT: v_writelane_b32 v22, s51, 9
-; SI-NEXT: v_writelane_b32 v22, s52, 10
-; SI-NEXT: v_writelane_b32 v22, s53, 11
-; SI-NEXT: v_writelane_b32 v22, s54, 12
-; SI-NEXT: v_writelane_b32 v22, s55, 13
-; SI-NEXT: v_writelane_b32 v22, s64, 14
-; SI-NEXT: v_writelane_b32 v22, s65, 15
-; SI-NEXT: v_writelane_b32 v22, s66, 16
-; SI-NEXT: v_writelane_b32 v22, s67, 17
-; SI-NEXT: v_writelane_b32 v22, s30, 18
-; SI-NEXT: v_writelane_b32 v22, s31, 19
+; SI-NEXT: v_writelane_b32 v22, s36, 0
+; SI-NEXT: v_writelane_b32 v22, s37, 1
+; SI-NEXT: v_writelane_b32 v22, s38, 2
+; SI-NEXT: v_writelane_b32 v22, s39, 3
+; SI-NEXT: v_writelane_b32 v22, s48, 4
+; SI-NEXT: v_writelane_b32 v22, s49, 5
+; SI-NEXT: v_writelane_b32 v22, s50, 6
+; SI-NEXT: v_writelane_b32 v22, s51, 7
+; SI-NEXT: v_writelane_b32 v22, s52, 8
+; SI-NEXT: v_writelane_b32 v22, s53, 9
+; SI-NEXT: v_writelane_b32 v22, s54, 10
+; SI-NEXT: v_writelane_b32 v22, s55, 11
+; SI-NEXT: v_writelane_b32 v22, s64, 12
+; SI-NEXT: v_writelane_b32 v22, s65, 13
+; SI-NEXT: v_writelane_b32 v22, s66, 14
+; SI-NEXT: v_writelane_b32 v22, s67, 15
+; SI-NEXT: v_writelane_b32 v22, s30, 16
+; SI-NEXT: v_writelane_b32 v22, s31, 17
; SI-NEXT: v_readfirstlane_b32 s7, v7
; SI-NEXT: v_readfirstlane_b32 s9, v6
; SI-NEXT: v_readfirstlane_b32 s11, v5
@@ -14241,10 +14549,10 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -14255,18 +14563,18 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s89, s90, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -14323,26 +14631,35 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -14435,8 +14752,8 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v22, 18
+; SI-NEXT: .LBB31_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v22, 16
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -14459,33 +14776,28 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v19, s55
; SI-NEXT: v_mov_b32_e32 v20, s56
; SI-NEXT: v_mov_b32_e32 v21, s57
-; SI-NEXT: v_readlane_b32 s31, v22, 19
-; SI-NEXT: v_readlane_b32 s67, v22, 17
-; SI-NEXT: v_readlane_b32 s66, v22, 16
-; SI-NEXT: v_readlane_b32 s65, v22, 15
-; SI-NEXT: v_readlane_b32 s64, v22, 14
-; SI-NEXT: v_readlane_b32 s55, v22, 13
-; SI-NEXT: v_readlane_b32 s54, v22, 12
-; SI-NEXT: v_readlane_b32 s53, v22, 11
-; SI-NEXT: v_readlane_b32 s52, v22, 10
-; SI-NEXT: v_readlane_b32 s51, v22, 9
-; SI-NEXT: v_readlane_b32 s50, v22, 8
-; SI-NEXT: v_readlane_b32 s49, v22, 7
-; SI-NEXT: v_readlane_b32 s48, v22, 6
-; SI-NEXT: v_readlane_b32 s39, v22, 5
-; SI-NEXT: v_readlane_b32 s38, v22, 4
-; SI-NEXT: v_readlane_b32 s37, v22, 3
-; SI-NEXT: v_readlane_b32 s36, v22, 2
-; SI-NEXT: v_readlane_b32 s35, v22, 1
-; SI-NEXT: v_readlane_b32 s34, v22, 0
+; SI-NEXT: v_readlane_b32 s31, v22, 17
+; SI-NEXT: v_readlane_b32 s67, v22, 15
+; SI-NEXT: v_readlane_b32 s66, v22, 14
+; SI-NEXT: v_readlane_b32 s65, v22, 13
+; SI-NEXT: v_readlane_b32 s64, v22, 12
+; SI-NEXT: v_readlane_b32 s55, v22, 11
+; SI-NEXT: v_readlane_b32 s54, v22, 10
+; SI-NEXT: v_readlane_b32 s53, v22, 9
+; SI-NEXT: v_readlane_b32 s52, v22, 8
+; SI-NEXT: v_readlane_b32 s51, v22, 7
+; SI-NEXT: v_readlane_b32 s50, v22, 6
+; SI-NEXT: v_readlane_b32 s49, v22, 5
+; SI-NEXT: v_readlane_b32 s48, v22, 4
+; SI-NEXT: v_readlane_b32 s39, v22, 3
+; SI-NEXT: v_readlane_b32 s38, v22, 2
+; SI-NEXT: v_readlane_b32 s37, v22, 1
+; SI-NEXT: v_readlane_b32 s36, v22, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v44i16_to_v22f32_scalar:
; VI: ; %bb.0:
@@ -14513,10 +14825,8 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v22, s67, 17
; VI-NEXT: v_writelane_b32 v22, s68, 18
; VI-NEXT: v_writelane_b32 v22, s69, 19
-; VI-NEXT: v_writelane_b32 v22, s70, 20
-; VI-NEXT: v_writelane_b32 v22, s71, 21
-; VI-NEXT: v_writelane_b32 v22, s30, 22
-; VI-NEXT: v_writelane_b32 v22, s31, 23
+; VI-NEXT: v_writelane_b32 v22, s30, 20
+; VI-NEXT: v_writelane_b32 v22, s31, 21
; VI-NEXT: v_readfirstlane_b32 s7, v7
; VI-NEXT: v_readfirstlane_b32 s9, v6
; VI-NEXT: v_readfirstlane_b32 s11, v5
@@ -14531,14 +14841,14 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s78, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -14549,31 +14859,31 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -14617,46 +14927,55 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -14729,8 +15048,8 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s57, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v22, 22
+; VI-NEXT: .LBB31_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v22, 20
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -14753,9 +15072,7 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v19, s55
; VI-NEXT: v_mov_b32_e32 v20, s56
; VI-NEXT: v_mov_b32_e32 v21, s57
-; VI-NEXT: v_readlane_b32 s31, v22, 23
-; VI-NEXT: v_readlane_b32 s71, v22, 21
-; VI-NEXT: v_readlane_b32 s70, v22, 20
+; VI-NEXT: v_readlane_b32 s31, v22, 21
; VI-NEXT: v_readlane_b32 s69, v22, 19
; VI-NEXT: v_readlane_b32 s68, v22, 18
; VI-NEXT: v_readlane_b32 s67, v22, 17
@@ -14781,9 +15098,6 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v44i16_to_v22f32_scalar:
; GFX9: ; %bb.0:
@@ -14857,10 +15171,18 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -14883,10 +15205,8 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v20, s56, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -14919,7 +15239,7 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -14992,11 +15312,16 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -15020,8 +15345,6 @@ define inreg <22 x float> @bitcast_v44i16_to_v22f32_scalar(<44 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v20, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v21, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -15675,7 +15998,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s94, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
@@ -15699,8 +16022,38 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v21, s5, 1.0
; SI-NEXT: v_add_f32_e64 v20, s4, 1.0
; SI-NEXT: v_add_f32_e64 v19, s7, 1.0
@@ -15745,32 +16098,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15815,7 +16144,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s60
; SI-NEXT: v_mov_b32_e32 v31, s62
; SI-NEXT: v_mov_b32_e32 v32, s72
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
@@ -15897,7 +16226,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -15921,8 +16250,38 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v21, s6, 1.0
; VI-NEXT: v_add_f32_e64 v20, s7, 1.0
; VI-NEXT: v_add_f32_e64 v19, s8, 1.0
@@ -15967,32 +16326,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16037,7 +16372,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s40
; VI-NEXT: v_mov_b32_e32 v23, s15
; VI-NEXT: v_mov_b32_e32 v22, s14
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; VI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
@@ -16097,7 +16432,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -16121,8 +16456,38 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v21, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v20, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v19, s8, 1.0
@@ -16167,32 +16532,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16237,7 +16578,7 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s40
; GFX9-NEXT: v_mov_b32_e32 v23, s15
; GFX9-NEXT: v_mov_b32_e32 v22, s14
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -16293,11 +16634,11 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s29, 16
@@ -16318,9 +16659,38 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v21, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v20, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s6, 1.0
@@ -16365,32 +16735,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -16412,8 +16758,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s9 :: v_dual_mov_b32 v22, s8
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v51.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v50.l
@@ -16448,11 +16794,11 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s29, 16
@@ -16473,9 +16819,38 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v17, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v18, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v19, s6, 1.0
@@ -16520,32 +16895,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v5, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v3, s17
@@ -16567,8 +16918,8 @@ define inreg <44 x half> @bitcast_v22f32_to_v44f16_scalar(<22 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s10 :: v_dual_mov_b32 v22, s9
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s9 :: v_dual_mov_b32 v22, s8
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v1
@@ -17686,26 +18037,24 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v32, s34, 0
-; SI-NEXT: v_writelane_b32 v32, s35, 1
-; SI-NEXT: v_writelane_b32 v32, s36, 2
-; SI-NEXT: v_writelane_b32 v32, s37, 3
-; SI-NEXT: v_writelane_b32 v32, s38, 4
-; SI-NEXT: v_writelane_b32 v32, s39, 5
-; SI-NEXT: v_writelane_b32 v32, s48, 6
-; SI-NEXT: v_writelane_b32 v32, s49, 7
-; SI-NEXT: v_writelane_b32 v32, s50, 8
-; SI-NEXT: v_writelane_b32 v32, s51, 9
-; SI-NEXT: v_writelane_b32 v32, s52, 10
-; SI-NEXT: v_writelane_b32 v32, s53, 11
-; SI-NEXT: v_writelane_b32 v32, s54, 12
-; SI-NEXT: v_writelane_b32 v32, s55, 13
-; SI-NEXT: v_writelane_b32 v32, s64, 14
-; SI-NEXT: v_writelane_b32 v32, s65, 15
-; SI-NEXT: v_writelane_b32 v32, s66, 16
-; SI-NEXT: v_writelane_b32 v32, s67, 17
-; SI-NEXT: v_writelane_b32 v32, s30, 18
-; SI-NEXT: v_writelane_b32 v32, s31, 19
+; SI-NEXT: v_writelane_b32 v32, s36, 0
+; SI-NEXT: v_writelane_b32 v32, s37, 1
+; SI-NEXT: v_writelane_b32 v32, s38, 2
+; SI-NEXT: v_writelane_b32 v32, s39, 3
+; SI-NEXT: v_writelane_b32 v32, s48, 4
+; SI-NEXT: v_writelane_b32 v32, s49, 5
+; SI-NEXT: v_writelane_b32 v32, s50, 6
+; SI-NEXT: v_writelane_b32 v32, s51, 7
+; SI-NEXT: v_writelane_b32 v32, s52, 8
+; SI-NEXT: v_writelane_b32 v32, s53, 9
+; SI-NEXT: v_writelane_b32 v32, s54, 10
+; SI-NEXT: v_writelane_b32 v32, s55, 11
+; SI-NEXT: v_writelane_b32 v32, s64, 12
+; SI-NEXT: v_writelane_b32 v32, s65, 13
+; SI-NEXT: v_writelane_b32 v32, s66, 14
+; SI-NEXT: v_writelane_b32 v32, s67, 15
+; SI-NEXT: v_writelane_b32 v32, s30, 16
+; SI-NEXT: v_writelane_b32 v32, s31, 17
; SI-NEXT: v_readfirstlane_b32 s6, v7
; SI-NEXT: v_readfirstlane_b32 s8, v6
; SI-NEXT: v_readfirstlane_b32 s10, v5
@@ -17724,10 +18073,10 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -17738,18 +18087,18 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s79, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -17806,11 +18155,20 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -17820,14 +18178,14 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_lo
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -17984,11 +18342,8 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v20, v21, v20
; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v22
; SI-NEXT: v_or_b32_e32 v21, v23, v21
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -18021,27 +18376,25 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 18
-; SI-NEXT: v_readlane_b32 s31, v32, 19
-; SI-NEXT: v_readlane_b32 s67, v32, 17
-; SI-NEXT: v_readlane_b32 s66, v32, 16
-; SI-NEXT: v_readlane_b32 s65, v32, 15
-; SI-NEXT: v_readlane_b32 s64, v32, 14
-; SI-NEXT: v_readlane_b32 s55, v32, 13
-; SI-NEXT: v_readlane_b32 s54, v32, 12
-; SI-NEXT: v_readlane_b32 s53, v32, 11
-; SI-NEXT: v_readlane_b32 s52, v32, 10
-; SI-NEXT: v_readlane_b32 s51, v32, 9
-; SI-NEXT: v_readlane_b32 s50, v32, 8
-; SI-NEXT: v_readlane_b32 s49, v32, 7
-; SI-NEXT: v_readlane_b32 s48, v32, 6
-; SI-NEXT: v_readlane_b32 s39, v32, 5
-; SI-NEXT: v_readlane_b32 s38, v32, 4
-; SI-NEXT: v_readlane_b32 s37, v32, 3
-; SI-NEXT: v_readlane_b32 s36, v32, 2
-; SI-NEXT: v_readlane_b32 s35, v32, 1
-; SI-NEXT: v_readlane_b32 s34, v32, 0
+; SI-NEXT: .LBB35_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 16
+; SI-NEXT: v_readlane_b32 s31, v32, 17
+; SI-NEXT: v_readlane_b32 s67, v32, 15
+; SI-NEXT: v_readlane_b32 s66, v32, 14
+; SI-NEXT: v_readlane_b32 s65, v32, 13
+; SI-NEXT: v_readlane_b32 s64, v32, 12
+; SI-NEXT: v_readlane_b32 s55, v32, 11
+; SI-NEXT: v_readlane_b32 s54, v32, 10
+; SI-NEXT: v_readlane_b32 s53, v32, 9
+; SI-NEXT: v_readlane_b32 s52, v32, 8
+; SI-NEXT: v_readlane_b32 s51, v32, 7
+; SI-NEXT: v_readlane_b32 s50, v32, 6
+; SI-NEXT: v_readlane_b32 s49, v32, 5
+; SI-NEXT: v_readlane_b32 s48, v32, 4
+; SI-NEXT: v_readlane_b32 s39, v32, 3
+; SI-NEXT: v_readlane_b32 s38, v32, 2
+; SI-NEXT: v_readlane_b32 s37, v32, 1
+; SI-NEXT: v_readlane_b32 s36, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -18074,10 +18427,8 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: v_writelane_b32 v32, s67, 17
; VI-NEXT: v_writelane_b32 v32, s68, 18
; VI-NEXT: v_writelane_b32 v32, s69, 19
-; VI-NEXT: v_writelane_b32 v32, s70, 20
-; VI-NEXT: v_writelane_b32 v32, s71, 21
-; VI-NEXT: v_writelane_b32 v32, s30, 22
-; VI-NEXT: v_writelane_b32 v32, s31, 23
+; VI-NEXT: v_writelane_b32 v32, s30, 20
+; VI-NEXT: v_writelane_b32 v32, s31, 21
; VI-NEXT: v_readfirstlane_b32 s6, v7
; VI-NEXT: v_readfirstlane_b32 s8, v6
; VI-NEXT: v_readfirstlane_b32 s10, v5
@@ -18092,14 +18443,14 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s77, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s90, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -18110,31 +18461,31 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s91, s89, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -18178,38 +18529,47 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v21, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s71
-; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v0, s69
+; VI-NEXT: v_mov_b32_e32 v2, s68
; VI-NEXT: v_add_f16_sdwa v0, v0, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v21
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v21
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s69
+; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v21
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v3, s34
; VI-NEXT: v_add_f16_sdwa v3, v3, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v21
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s35
+; VI-NEXT: v_mov_b32_e32 v4, s31
; VI-NEXT: v_add_f16_sdwa v4, v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v21
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s34
+; VI-NEXT: v_mov_b32_e32 v5, s30
; VI-NEXT: v_add_f16_sdwa v5, v5, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v21
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s31
+; VI-NEXT: v_mov_b32_e32 v6, vcc_hi
; VI-NEXT: v_add_f16_sdwa v6, v6, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v21
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s30
+; VI-NEXT: v_mov_b32_e32 v7, vcc_lo
; VI-NEXT: v_add_f16_sdwa v7, v7, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v21
; VI-NEXT: v_or_b32_e32 v7, v8, v7
@@ -18269,11 +18629,8 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v22, v22, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s6, v21
; VI-NEXT: v_or_b32_e32 v21, v21, v22
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -18306,11 +18663,9 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 22
-; VI-NEXT: v_readlane_b32 s31, v32, 23
-; VI-NEXT: v_readlane_b32 s71, v32, 21
-; VI-NEXT: v_readlane_b32 s70, v32, 20
+; VI-NEXT: .LBB35_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 20
+; VI-NEXT: v_readlane_b32 s31, v32, 21
; VI-NEXT: v_readlane_b32 s69, v32, 19
; VI-NEXT: v_readlane_b32 s68, v32, 18
; VI-NEXT: v_readlane_b32 s67, v32, 17
@@ -18409,10 +18764,18 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v21, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v21 op_sel_hi:[1,0]
@@ -18436,10 +18799,8 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v19, s55, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v20, s56, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v21, s57, v21 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -18472,7 +18833,7 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -18545,11 +18906,16 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -18573,8 +18939,6 @@ define inreg <22 x float> @bitcast_v44f16_to_v22f32_scalar(<44 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v20, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v21, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -18788,10 +19152,18 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s12, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s13, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -18814,7 +19186,7 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -18838,8 +19210,6 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v20, s7
; SI-NEXT: v_mov_b32_e32 v21, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v11i64_to_v11f64_scalar:
; VI: ; %bb.0:
@@ -18854,10 +19224,18 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -18880,7 +19258,7 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -18904,8 +19282,6 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v11i64_to_v11f64_scalar:
; GFX9: ; %bb.0:
@@ -18920,10 +19296,18 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -18946,7 +19330,7 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -18970,8 +19354,6 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v11i64_to_v11f64_scalar:
; GFX11: ; %bb.0:
@@ -18983,11 +19365,16 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s8, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -19010,7 +19397,7 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -19023,8 +19410,6 @@ define inreg <11 x double> @bitcast_v11i64_to_v11f64_scalar(<11 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -19200,10 +19585,18 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -19215,10 +19608,8 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; SI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -19251,7 +19642,7 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -19312,10 +19703,18 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -19327,10 +19726,8 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; VI-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -19363,7 +19760,7 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -19424,10 +19821,18 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -19439,10 +19844,8 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[16:17], s[52:53], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -19475,7 +19878,7 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -19537,11 +19940,16 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -19554,8 +19962,6 @@ define inreg <11 x i64> @bitcast_v11f64_to_v11i64_scalar(<11 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -20260,7 +20666,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s77, s7, 16
@@ -20284,8 +20690,38 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -20330,7 +20766,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s13, s72, 16
; SI-NEXT: s_and_b32 s15, s16, 0xffff
; SI-NEXT: s_or_b32 s13, s15, s13
@@ -20420,30 +20856,6 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v20, s4
; SI-NEXT: v_mov_b32_e32 v21, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v11i64_to_v44i16_scalar:
; VI: ; %bb.0:
@@ -20458,7 +20870,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -20482,8 +20894,38 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -20528,7 +20970,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -20618,30 +21060,6 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v11i64_to_v44i16_scalar:
; GFX9: ; %bb.0:
@@ -20656,7 +21074,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -20680,8 +21098,38 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -20726,7 +21174,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s73
@@ -20772,30 +21220,6 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v11i64_to_v44i16_scalar:
; GFX11: ; %bb.0:
@@ -20807,7 +21231,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s62, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
; GFX11-NEXT: s_lshr_b32 s9, s5, 16
@@ -20831,9 +21255,38 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s62, -1
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -20878,7 +21331,7 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s60
@@ -20914,30 +21367,6 @@ define inreg <44 x i16> @bitcast_v11i64_to_v44i16_scalar(<11 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -21911,26 +22340,24 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s34, 0
-; SI-NEXT: v_writelane_b32 v22, s35, 1
-; SI-NEXT: v_writelane_b32 v22, s36, 2
-; SI-NEXT: v_writelane_b32 v22, s37, 3
-; SI-NEXT: v_writelane_b32 v22, s38, 4
-; SI-NEXT: v_writelane_b32 v22, s39, 5
-; SI-NEXT: v_writelane_b32 v22, s48, 6
-; SI-NEXT: v_writelane_b32 v22, s49, 7
-; SI-NEXT: v_writelane_b32 v22, s50, 8
-; SI-NEXT: v_writelane_b32 v22, s51, 9
-; SI-NEXT: v_writelane_b32 v22, s52, 10
-; SI-NEXT: v_writelane_b32 v22, s53, 11
-; SI-NEXT: v_writelane_b32 v22, s54, 12
-; SI-NEXT: v_writelane_b32 v22, s55, 13
-; SI-NEXT: v_writelane_b32 v22, s64, 14
-; SI-NEXT: v_writelane_b32 v22, s65, 15
-; SI-NEXT: v_writelane_b32 v22, s66, 16
-; SI-NEXT: v_writelane_b32 v22, s67, 17
-; SI-NEXT: v_writelane_b32 v22, s30, 18
-; SI-NEXT: v_writelane_b32 v22, s31, 19
+; SI-NEXT: v_writelane_b32 v22, s36, 0
+; SI-NEXT: v_writelane_b32 v22, s37, 1
+; SI-NEXT: v_writelane_b32 v22, s38, 2
+; SI-NEXT: v_writelane_b32 v22, s39, 3
+; SI-NEXT: v_writelane_b32 v22, s48, 4
+; SI-NEXT: v_writelane_b32 v22, s49, 5
+; SI-NEXT: v_writelane_b32 v22, s50, 6
+; SI-NEXT: v_writelane_b32 v22, s51, 7
+; SI-NEXT: v_writelane_b32 v22, s52, 8
+; SI-NEXT: v_writelane_b32 v22, s53, 9
+; SI-NEXT: v_writelane_b32 v22, s54, 10
+; SI-NEXT: v_writelane_b32 v22, s55, 11
+; SI-NEXT: v_writelane_b32 v22, s64, 12
+; SI-NEXT: v_writelane_b32 v22, s65, 13
+; SI-NEXT: v_writelane_b32 v22, s66, 14
+; SI-NEXT: v_writelane_b32 v22, s67, 15
+; SI-NEXT: v_writelane_b32 v22, s30, 16
+; SI-NEXT: v_writelane_b32 v22, s31, 17
; SI-NEXT: v_readfirstlane_b32 s7, v7
; SI-NEXT: v_readfirstlane_b32 s9, v6
; SI-NEXT: v_readfirstlane_b32 s11, v5
@@ -21949,10 +22376,10 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -21963,18 +22390,18 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s89, s90, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -22031,26 +22458,35 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -22143,8 +22579,8 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v22, 18
+; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v22, 16
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -22167,33 +22603,28 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v19, s55
; SI-NEXT: v_mov_b32_e32 v20, s56
; SI-NEXT: v_mov_b32_e32 v21, s57
-; SI-NEXT: v_readlane_b32 s31, v22, 19
-; SI-NEXT: v_readlane_b32 s67, v22, 17
-; SI-NEXT: v_readlane_b32 s66, v22, 16
-; SI-NEXT: v_readlane_b32 s65, v22, 15
-; SI-NEXT: v_readlane_b32 s64, v22, 14
-; SI-NEXT: v_readlane_b32 s55, v22, 13
-; SI-NEXT: v_readlane_b32 s54, v22, 12
-; SI-NEXT: v_readlane_b32 s53, v22, 11
-; SI-NEXT: v_readlane_b32 s52, v22, 10
-; SI-NEXT: v_readlane_b32 s51, v22, 9
-; SI-NEXT: v_readlane_b32 s50, v22, 8
-; SI-NEXT: v_readlane_b32 s49, v22, 7
-; SI-NEXT: v_readlane_b32 s48, v22, 6
-; SI-NEXT: v_readlane_b32 s39, v22, 5
-; SI-NEXT: v_readlane_b32 s38, v22, 4
-; SI-NEXT: v_readlane_b32 s37, v22, 3
-; SI-NEXT: v_readlane_b32 s36, v22, 2
-; SI-NEXT: v_readlane_b32 s35, v22, 1
-; SI-NEXT: v_readlane_b32 s34, v22, 0
+; SI-NEXT: v_readlane_b32 s31, v22, 17
+; SI-NEXT: v_readlane_b32 s67, v22, 15
+; SI-NEXT: v_readlane_b32 s66, v22, 14
+; SI-NEXT: v_readlane_b32 s65, v22, 13
+; SI-NEXT: v_readlane_b32 s64, v22, 12
+; SI-NEXT: v_readlane_b32 s55, v22, 11
+; SI-NEXT: v_readlane_b32 s54, v22, 10
+; SI-NEXT: v_readlane_b32 s53, v22, 9
+; SI-NEXT: v_readlane_b32 s52, v22, 8
+; SI-NEXT: v_readlane_b32 s51, v22, 7
+; SI-NEXT: v_readlane_b32 s50, v22, 6
+; SI-NEXT: v_readlane_b32 s49, v22, 5
+; SI-NEXT: v_readlane_b32 s48, v22, 4
+; SI-NEXT: v_readlane_b32 s39, v22, 3
+; SI-NEXT: v_readlane_b32 s38, v22, 2
+; SI-NEXT: v_readlane_b32 s37, v22, 1
+; SI-NEXT: v_readlane_b32 s36, v22, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v44i16_to_v11i64_scalar:
; VI: ; %bb.0:
@@ -22221,10 +22652,8 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v22, s67, 17
; VI-NEXT: v_writelane_b32 v22, s68, 18
; VI-NEXT: v_writelane_b32 v22, s69, 19
-; VI-NEXT: v_writelane_b32 v22, s70, 20
-; VI-NEXT: v_writelane_b32 v22, s71, 21
-; VI-NEXT: v_writelane_b32 v22, s30, 22
-; VI-NEXT: v_writelane_b32 v22, s31, 23
+; VI-NEXT: v_writelane_b32 v22, s30, 20
+; VI-NEXT: v_writelane_b32 v22, s31, 21
; VI-NEXT: v_readfirstlane_b32 s7, v7
; VI-NEXT: v_readfirstlane_b32 s9, v6
; VI-NEXT: v_readfirstlane_b32 s11, v5
@@ -22239,14 +22668,14 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s78, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -22257,31 +22686,31 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -22325,46 +22754,55 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -22437,8 +22875,8 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s57, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v22, 22
+; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v22, 20
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -22461,9 +22899,7 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v19, s55
; VI-NEXT: v_mov_b32_e32 v20, s56
; VI-NEXT: v_mov_b32_e32 v21, s57
-; VI-NEXT: v_readlane_b32 s31, v22, 23
-; VI-NEXT: v_readlane_b32 s71, v22, 21
-; VI-NEXT: v_readlane_b32 s70, v22, 20
+; VI-NEXT: v_readlane_b32 s31, v22, 21
; VI-NEXT: v_readlane_b32 s69, v22, 19
; VI-NEXT: v_readlane_b32 s68, v22, 18
; VI-NEXT: v_readlane_b32 s67, v22, 17
@@ -22489,9 +22925,6 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v44i16_to_v11i64_scalar:
; GFX9: ; %bb.0:
@@ -22565,10 +22998,18 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -22591,10 +23032,8 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v20, s56, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -22627,7 +23066,7 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -22700,11 +23139,16 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -22728,8 +23172,6 @@ define inreg <11 x i64> @bitcast_v44i16_to_v11i64_scalar(<44 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v20, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v21, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -23417,7 +23859,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s11, v1
; SI-NEXT: s_cmp_lg_u32 s10, 0
; SI-NEXT: v_readfirstlane_b32 s10, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s76, s5, 16
; SI-NEXT: s_lshr_b32 s77, s7, 16
@@ -23441,8 +23883,38 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -23487,7 +23959,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s13, s72, 16
; SI-NEXT: s_and_b32 s15, s16, 0xffff
; SI-NEXT: s_or_b32 s13, s15, s13
@@ -23577,30 +24049,6 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v20, s4
; SI-NEXT: v_mov_b32_e32 v21, s5
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v11i64_to_v44f16_scalar:
; VI: ; %bb.0:
@@ -23615,7 +24063,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s12, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s13, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s6, 16
; VI-NEXT: s_lshr_b32 s15, s7, 16
@@ -23639,8 +24087,38 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -23685,7 +24163,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s18, 16
; VI-NEXT: s_lshr_b32 s74, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -23775,30 +24253,6 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v20, s7
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v11i64_to_v44f16_scalar:
; GFX9: ; %bb.0:
@@ -23813,7 +24267,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s12, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s6, 16
; GFX9-NEXT: s_lshr_b32 s15, s7, 16
@@ -23837,8 +24291,38 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -23883,7 +24367,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s73, s18, 16
; GFX9-NEXT: s_lshr_b32 s74, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s73
@@ -23929,30 +24413,6 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v20, s7
; GFX9-NEXT: v_mov_b32_e32 v21, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v11i64_to_v44f16_scalar:
; GFX11: ; %bb.0:
@@ -23964,7 +24424,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s7, v0
; GFX11-NEXT: s_cmp_lg_u32 s8, 0
; GFX11-NEXT: s_mov_b32 s62, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s8, s4, 16
; GFX11-NEXT: s_lshr_b32 s9, s5, 16
@@ -23988,9 +24448,38 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s62, -1
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -24035,7 +24524,7 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s59, s2, 16
; GFX11-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-NEXT: s_lshr_b32 s61, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s60
@@ -24071,30 +24560,6 @@ define inreg <44 x half> @bitcast_v11i64_to_v44f16_scalar(<11 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v18, s7 :: v_dual_mov_b32 v19, s6
; GFX11-NEXT: v_dual_mov_b32 v20, s5 :: v_dual_mov_b32 v21, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -25167,26 +25632,24 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v32, s34, 0
-; SI-NEXT: v_writelane_b32 v32, s35, 1
-; SI-NEXT: v_writelane_b32 v32, s36, 2
-; SI-NEXT: v_writelane_b32 v32, s37, 3
-; SI-NEXT: v_writelane_b32 v32, s38, 4
-; SI-NEXT: v_writelane_b32 v32, s39, 5
-; SI-NEXT: v_writelane_b32 v32, s48, 6
-; SI-NEXT: v_writelane_b32 v32, s49, 7
-; SI-NEXT: v_writelane_b32 v32, s50, 8
-; SI-NEXT: v_writelane_b32 v32, s51, 9
-; SI-NEXT: v_writelane_b32 v32, s52, 10
-; SI-NEXT: v_writelane_b32 v32, s53, 11
-; SI-NEXT: v_writelane_b32 v32, s54, 12
-; SI-NEXT: v_writelane_b32 v32, s55, 13
-; SI-NEXT: v_writelane_b32 v32, s64, 14
-; SI-NEXT: v_writelane_b32 v32, s65, 15
-; SI-NEXT: v_writelane_b32 v32, s66, 16
-; SI-NEXT: v_writelane_b32 v32, s67, 17
-; SI-NEXT: v_writelane_b32 v32, s30, 18
-; SI-NEXT: v_writelane_b32 v32, s31, 19
+; SI-NEXT: v_writelane_b32 v32, s36, 0
+; SI-NEXT: v_writelane_b32 v32, s37, 1
+; SI-NEXT: v_writelane_b32 v32, s38, 2
+; SI-NEXT: v_writelane_b32 v32, s39, 3
+; SI-NEXT: v_writelane_b32 v32, s48, 4
+; SI-NEXT: v_writelane_b32 v32, s49, 5
+; SI-NEXT: v_writelane_b32 v32, s50, 6
+; SI-NEXT: v_writelane_b32 v32, s51, 7
+; SI-NEXT: v_writelane_b32 v32, s52, 8
+; SI-NEXT: v_writelane_b32 v32, s53, 9
+; SI-NEXT: v_writelane_b32 v32, s54, 10
+; SI-NEXT: v_writelane_b32 v32, s55, 11
+; SI-NEXT: v_writelane_b32 v32, s64, 12
+; SI-NEXT: v_writelane_b32 v32, s65, 13
+; SI-NEXT: v_writelane_b32 v32, s66, 14
+; SI-NEXT: v_writelane_b32 v32, s67, 15
+; SI-NEXT: v_writelane_b32 v32, s30, 16
+; SI-NEXT: v_writelane_b32 v32, s31, 17
; SI-NEXT: v_readfirstlane_b32 s6, v7
; SI-NEXT: v_readfirstlane_b32 s8, v6
; SI-NEXT: v_readfirstlane_b32 s10, v5
@@ -25205,10 +25668,10 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -25219,18 +25682,18 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s79, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -25287,11 +25750,20 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -25301,14 +25773,14 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_lo
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -25465,11 +25937,8 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v20, v21, v20
; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v22
; SI-NEXT: v_or_b32_e32 v21, v23, v21
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -25502,27 +25971,25 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 18
-; SI-NEXT: v_readlane_b32 s31, v32, 19
-; SI-NEXT: v_readlane_b32 s67, v32, 17
-; SI-NEXT: v_readlane_b32 s66, v32, 16
-; SI-NEXT: v_readlane_b32 s65, v32, 15
-; SI-NEXT: v_readlane_b32 s64, v32, 14
-; SI-NEXT: v_readlane_b32 s55, v32, 13
-; SI-NEXT: v_readlane_b32 s54, v32, 12
-; SI-NEXT: v_readlane_b32 s53, v32, 11
-; SI-NEXT: v_readlane_b32 s52, v32, 10
-; SI-NEXT: v_readlane_b32 s51, v32, 9
-; SI-NEXT: v_readlane_b32 s50, v32, 8
-; SI-NEXT: v_readlane_b32 s49, v32, 7
-; SI-NEXT: v_readlane_b32 s48, v32, 6
-; SI-NEXT: v_readlane_b32 s39, v32, 5
-; SI-NEXT: v_readlane_b32 s38, v32, 4
-; SI-NEXT: v_readlane_b32 s37, v32, 3
-; SI-NEXT: v_readlane_b32 s36, v32, 2
-; SI-NEXT: v_readlane_b32 s35, v32, 1
-; SI-NEXT: v_readlane_b32 s34, v32, 0
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 16
+; SI-NEXT: v_readlane_b32 s31, v32, 17
+; SI-NEXT: v_readlane_b32 s67, v32, 15
+; SI-NEXT: v_readlane_b32 s66, v32, 14
+; SI-NEXT: v_readlane_b32 s65, v32, 13
+; SI-NEXT: v_readlane_b32 s64, v32, 12
+; SI-NEXT: v_readlane_b32 s55, v32, 11
+; SI-NEXT: v_readlane_b32 s54, v32, 10
+; SI-NEXT: v_readlane_b32 s53, v32, 9
+; SI-NEXT: v_readlane_b32 s52, v32, 8
+; SI-NEXT: v_readlane_b32 s51, v32, 7
+; SI-NEXT: v_readlane_b32 s50, v32, 6
+; SI-NEXT: v_readlane_b32 s49, v32, 5
+; SI-NEXT: v_readlane_b32 s48, v32, 4
+; SI-NEXT: v_readlane_b32 s39, v32, 3
+; SI-NEXT: v_readlane_b32 s38, v32, 2
+; SI-NEXT: v_readlane_b32 s37, v32, 1
+; SI-NEXT: v_readlane_b32 s36, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -25555,10 +26022,8 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s67, 17
; VI-NEXT: v_writelane_b32 v32, s68, 18
; VI-NEXT: v_writelane_b32 v32, s69, 19
-; VI-NEXT: v_writelane_b32 v32, s70, 20
-; VI-NEXT: v_writelane_b32 v32, s71, 21
-; VI-NEXT: v_writelane_b32 v32, s30, 22
-; VI-NEXT: v_writelane_b32 v32, s31, 23
+; VI-NEXT: v_writelane_b32 v32, s30, 20
+; VI-NEXT: v_writelane_b32 v32, s31, 21
; VI-NEXT: v_readfirstlane_b32 s6, v7
; VI-NEXT: v_readfirstlane_b32 s8, v6
; VI-NEXT: v_readfirstlane_b32 s10, v5
@@ -25573,14 +26038,14 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s90, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -25591,31 +26056,31 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s91, s89, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -25659,38 +26124,47 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v21, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s71
-; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v0, s69
+; VI-NEXT: v_mov_b32_e32 v2, s68
; VI-NEXT: v_add_f16_sdwa v0, v0, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v21
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v21
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s69
+; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v21
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v3, s34
; VI-NEXT: v_add_f16_sdwa v3, v3, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v21
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s35
+; VI-NEXT: v_mov_b32_e32 v4, s31
; VI-NEXT: v_add_f16_sdwa v4, v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v21
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s34
+; VI-NEXT: v_mov_b32_e32 v5, s30
; VI-NEXT: v_add_f16_sdwa v5, v5, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v21
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s31
+; VI-NEXT: v_mov_b32_e32 v6, vcc_hi
; VI-NEXT: v_add_f16_sdwa v6, v6, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v21
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s30
+; VI-NEXT: v_mov_b32_e32 v7, vcc_lo
; VI-NEXT: v_add_f16_sdwa v7, v7, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v21
; VI-NEXT: v_or_b32_e32 v7, v8, v7
@@ -25750,11 +26224,8 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v22, v22, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s6, v21
; VI-NEXT: v_or_b32_e32 v21, v21, v22
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -25787,11 +26258,9 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 22
-; VI-NEXT: v_readlane_b32 s31, v32, 23
-; VI-NEXT: v_readlane_b32 s71, v32, 21
-; VI-NEXT: v_readlane_b32 s70, v32, 20
+; VI-NEXT: .LBB47_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 20
+; VI-NEXT: v_readlane_b32 s31, v32, 21
; VI-NEXT: v_readlane_b32 s69, v32, 19
; VI-NEXT: v_readlane_b32 s68, v32, 18
; VI-NEXT: v_readlane_b32 s67, v32, 17
@@ -25890,10 +26359,18 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v21, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v21 op_sel_hi:[1,0]
@@ -25917,10 +26394,8 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v19, s55, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v20, s56, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v21, s57, v21 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -25953,7 +26428,7 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -26026,11 +26501,16 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -26054,8 +26534,6 @@ define inreg <11 x i64> @bitcast_v44f16_to_v11i64_scalar(<44 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v20, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v21, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -26676,7 +27154,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s93, s11, 16
@@ -26700,8 +27178,38 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
@@ -26735,32 +27243,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -26805,7 +27289,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s40
; SI-NEXT: v_mov_b32_e32 v23, s14
; SI-NEXT: v_mov_b32_e32 v22, s12
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
@@ -26887,7 +27371,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s9, 16
; VI-NEXT: s_lshr_b32 s57, s8, 16
@@ -26911,8 +27395,38 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s74, s18, 16
; VI-NEXT: s_lshr_b32 s56, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s12, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s12, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
@@ -26946,32 +27460,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -27016,7 +27506,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s40
; VI-NEXT: v_mov_b32_e32 v26, s15
; VI-NEXT: v_mov_b32_e32 v24, s14
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -27076,7 +27566,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s9, 16
; GFX9-NEXT: s_lshr_b32 s57, s8, 16
@@ -27100,8 +27590,38 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s74, s18, 16
; GFX9-NEXT: s_lshr_b32 s56, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s12, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
@@ -27135,32 +27655,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -27205,7 +27701,7 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s40
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -27261,12 +27757,12 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s28, 16
@@ -27286,9 +27782,38 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
@@ -27322,32 +27847,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -27368,9 +27869,9 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s56 :: v_dual_mov_b32 v32, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s47 :: v_dual_mov_b32 v30, s12
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s46 :: v_dual_mov_b32 v28, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s45 :: v_dual_mov_b32 v26, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s43 :: v_dual_mov_b32 v24, s9
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s45 :: v_dual_mov_b32 v26, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s43 :: v_dual_mov_b32 v24, s8
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v51.l
@@ -27405,12 +27906,12 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s28, 16
@@ -27430,9 +27931,38 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[6:7], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[28:29], 1.0
@@ -27466,32 +27996,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v23, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v9, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v7, s22
@@ -27513,8 +28019,8 @@ define inreg <44 x i16> @bitcast_v11f64_to_v44i16_scalar(<11 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, s41 :: v_dual_mov_b32 v35, s14
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s15 :: v_dual_mov_b32 v33, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s12 :: v_dual_mov_b32 v29, s11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s10 :: v_dual_mov_b32 v27, s9
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s9 :: v_dual_mov_b32 v27, s8
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v23, 0xffff, v23
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
@@ -28533,26 +29039,24 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v22, s34, 0
-; SI-NEXT: v_writelane_b32 v22, s35, 1
-; SI-NEXT: v_writelane_b32 v22, s36, 2
-; SI-NEXT: v_writelane_b32 v22, s37, 3
-; SI-NEXT: v_writelane_b32 v22, s38, 4
-; SI-NEXT: v_writelane_b32 v22, s39, 5
-; SI-NEXT: v_writelane_b32 v22, s48, 6
-; SI-NEXT: v_writelane_b32 v22, s49, 7
-; SI-NEXT: v_writelane_b32 v22, s50, 8
-; SI-NEXT: v_writelane_b32 v22, s51, 9
-; SI-NEXT: v_writelane_b32 v22, s52, 10
-; SI-NEXT: v_writelane_b32 v22, s53, 11
-; SI-NEXT: v_writelane_b32 v22, s54, 12
-; SI-NEXT: v_writelane_b32 v22, s55, 13
-; SI-NEXT: v_writelane_b32 v22, s64, 14
-; SI-NEXT: v_writelane_b32 v22, s65, 15
-; SI-NEXT: v_writelane_b32 v22, s66, 16
-; SI-NEXT: v_writelane_b32 v22, s67, 17
-; SI-NEXT: v_writelane_b32 v22, s30, 18
-; SI-NEXT: v_writelane_b32 v22, s31, 19
+; SI-NEXT: v_writelane_b32 v22, s36, 0
+; SI-NEXT: v_writelane_b32 v22, s37, 1
+; SI-NEXT: v_writelane_b32 v22, s38, 2
+; SI-NEXT: v_writelane_b32 v22, s39, 3
+; SI-NEXT: v_writelane_b32 v22, s48, 4
+; SI-NEXT: v_writelane_b32 v22, s49, 5
+; SI-NEXT: v_writelane_b32 v22, s50, 6
+; SI-NEXT: v_writelane_b32 v22, s51, 7
+; SI-NEXT: v_writelane_b32 v22, s52, 8
+; SI-NEXT: v_writelane_b32 v22, s53, 9
+; SI-NEXT: v_writelane_b32 v22, s54, 10
+; SI-NEXT: v_writelane_b32 v22, s55, 11
+; SI-NEXT: v_writelane_b32 v22, s64, 12
+; SI-NEXT: v_writelane_b32 v22, s65, 13
+; SI-NEXT: v_writelane_b32 v22, s66, 14
+; SI-NEXT: v_writelane_b32 v22, s67, 15
+; SI-NEXT: v_writelane_b32 v22, s30, 16
+; SI-NEXT: v_writelane_b32 v22, s31, 17
; SI-NEXT: v_readfirstlane_b32 s7, v7
; SI-NEXT: v_readfirstlane_b32 s9, v6
; SI-NEXT: v_readfirstlane_b32 s11, v5
@@ -28571,10 +29075,10 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -28585,18 +29089,18 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s89, s90, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -28653,26 +29157,35 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
@@ -28765,8 +29278,8 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v22, 18
+; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v22, 16
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -28789,33 +29302,28 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v19, s55
; SI-NEXT: v_mov_b32_e32 v20, s56
; SI-NEXT: v_mov_b32_e32 v21, s57
-; SI-NEXT: v_readlane_b32 s31, v22, 19
-; SI-NEXT: v_readlane_b32 s67, v22, 17
-; SI-NEXT: v_readlane_b32 s66, v22, 16
-; SI-NEXT: v_readlane_b32 s65, v22, 15
-; SI-NEXT: v_readlane_b32 s64, v22, 14
-; SI-NEXT: v_readlane_b32 s55, v22, 13
-; SI-NEXT: v_readlane_b32 s54, v22, 12
-; SI-NEXT: v_readlane_b32 s53, v22, 11
-; SI-NEXT: v_readlane_b32 s52, v22, 10
-; SI-NEXT: v_readlane_b32 s51, v22, 9
-; SI-NEXT: v_readlane_b32 s50, v22, 8
-; SI-NEXT: v_readlane_b32 s49, v22, 7
-; SI-NEXT: v_readlane_b32 s48, v22, 6
-; SI-NEXT: v_readlane_b32 s39, v22, 5
-; SI-NEXT: v_readlane_b32 s38, v22, 4
-; SI-NEXT: v_readlane_b32 s37, v22, 3
-; SI-NEXT: v_readlane_b32 s36, v22, 2
-; SI-NEXT: v_readlane_b32 s35, v22, 1
-; SI-NEXT: v_readlane_b32 s34, v22, 0
+; SI-NEXT: v_readlane_b32 s31, v22, 17
+; SI-NEXT: v_readlane_b32 s67, v22, 15
+; SI-NEXT: v_readlane_b32 s66, v22, 14
+; SI-NEXT: v_readlane_b32 s65, v22, 13
+; SI-NEXT: v_readlane_b32 s64, v22, 12
+; SI-NEXT: v_readlane_b32 s55, v22, 11
+; SI-NEXT: v_readlane_b32 s54, v22, 10
+; SI-NEXT: v_readlane_b32 s53, v22, 9
+; SI-NEXT: v_readlane_b32 s52, v22, 8
+; SI-NEXT: v_readlane_b32 s51, v22, 7
+; SI-NEXT: v_readlane_b32 s50, v22, 6
+; SI-NEXT: v_readlane_b32 s49, v22, 5
+; SI-NEXT: v_readlane_b32 s48, v22, 4
+; SI-NEXT: v_readlane_b32 s39, v22, 3
+; SI-NEXT: v_readlane_b32 s38, v22, 2
+; SI-NEXT: v_readlane_b32 s37, v22, 1
+; SI-NEXT: v_readlane_b32 s36, v22, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v44i16_to_v11f64_scalar:
; VI: ; %bb.0:
@@ -28843,10 +29351,8 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v22, s67, 17
; VI-NEXT: v_writelane_b32 v22, s68, 18
; VI-NEXT: v_writelane_b32 v22, s69, 19
-; VI-NEXT: v_writelane_b32 v22, s70, 20
-; VI-NEXT: v_writelane_b32 v22, s71, 21
-; VI-NEXT: v_writelane_b32 v22, s30, 22
-; VI-NEXT: v_writelane_b32 v22, s31, 23
+; VI-NEXT: v_writelane_b32 v22, s30, 20
+; VI-NEXT: v_writelane_b32 v22, s31, 21
; VI-NEXT: v_readfirstlane_b32 s7, v7
; VI-NEXT: v_readfirstlane_b32 s9, v6
; VI-NEXT: v_readfirstlane_b32 s11, v5
@@ -28861,14 +29367,14 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s78, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -28879,31 +29385,31 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -28947,46 +29453,55 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -29059,8 +29574,8 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s57, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v22, 22
+; VI-NEXT: .LBB51_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v22, 20
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -29083,9 +29598,7 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v19, s55
; VI-NEXT: v_mov_b32_e32 v20, s56
; VI-NEXT: v_mov_b32_e32 v21, s57
-; VI-NEXT: v_readlane_b32 s31, v22, 23
-; VI-NEXT: v_readlane_b32 s71, v22, 21
-; VI-NEXT: v_readlane_b32 s70, v22, 20
+; VI-NEXT: v_readlane_b32 s31, v22, 21
; VI-NEXT: v_readlane_b32 s69, v22, 19
; VI-NEXT: v_readlane_b32 s68, v22, 18
; VI-NEXT: v_readlane_b32 s67, v22, 17
@@ -29111,9 +29624,6 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v44i16_to_v11f64_scalar:
; GFX9: ; %bb.0:
@@ -29187,10 +29697,18 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -29213,10 +29731,8 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v19, s55, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v20, s56, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -29249,7 +29765,7 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -29322,11 +29838,16 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -29350,8 +29871,6 @@ define inreg <11 x double> @bitcast_v44i16_to_v11f64_scalar(<44 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v20, s20, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v21, s21, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -29972,7 +30491,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s93, s11, 16
@@ -29996,8 +30515,38 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[60:61], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[62:63], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[74:75], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[74:75], -1
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr77
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr79
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[74:75], s[74:75], exec
+; SI-NEXT: s_cselect_b32 s13, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s13, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
; SI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
@@ -30031,32 +30580,8 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr77
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr79
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -30101,7 +30626,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v24, s40
; SI-NEXT: v_mov_b32_e32 v23, s14
; SI-NEXT: v_mov_b32_e32 v22, s12
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
@@ -30183,7 +30708,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s14, s9, 16
; VI-NEXT: s_lshr_b32 s57, s8, 16
@@ -30207,43 +30732,10 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; VI-NEXT: s_lshr_b32 s74, s18, 16
; VI-NEXT: s_lshr_b32 s56, s17, 16
; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
-; VI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
-; VI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
-; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
-; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
-; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
-; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
-; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
-; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
-; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
-; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; VI-NEXT: v_lshrrev_b32_e32 v24, 16, v21
-; VI-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; VI-NEXT: v_lshrrev_b32_e32 v26, 16, v19
-; VI-NEXT: v_lshrrev_b32_e32 v23, 16, v18
-; VI-NEXT: v_lshrrev_b32_e32 v28, 16, v17
-; VI-NEXT: v_lshrrev_b32_e32 v25, 16, v16
-; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v15
-; VI-NEXT: v_lshrrev_b32_e32 v27, 16, v14
-; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v13
-; VI-NEXT: v_lshrrev_b32_e32 v29, 16, v12
-; VI-NEXT: v_lshrrev_b32_e32 v34, 16, v11
-; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v10
-; VI-NEXT: v_lshrrev_b32_e32 v36, 16, v9
-; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v8
-; VI-NEXT: v_lshrrev_b32_e32 v38, 16, v7
-; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v6
-; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v5
-; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v4
-; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v3
-; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v2
-; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
-; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
+; VI-NEXT: s_mov_b64 s[12:13], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[12:13], -1
; VI-NEXT: ; implicit-def: $sgpr75
; VI-NEXT: ; implicit-def: $sgpr56
; VI-NEXT: ; implicit-def: $sgpr74
@@ -30266,8 +30758,47 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; VI-NEXT: ; implicit-def: $sgpr15
; VI-NEXT: ; implicit-def: $sgpr57
; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; VI-NEXT: s_cselect_b32 s12, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s12, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
+; VI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
+; VI-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
+; VI-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
+; VI-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
+; VI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
+; VI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
+; VI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; VI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; VI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; VI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; VI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; VI-NEXT: v_lshrrev_b32_e32 v24, 16, v21
+; VI-NEXT: v_lshrrev_b32_e32 v22, 16, v20
+; VI-NEXT: v_lshrrev_b32_e32 v26, 16, v19
+; VI-NEXT: v_lshrrev_b32_e32 v23, 16, v18
+; VI-NEXT: v_lshrrev_b32_e32 v28, 16, v17
+; VI-NEXT: v_lshrrev_b32_e32 v25, 16, v16
+; VI-NEXT: v_lshrrev_b32_e32 v30, 16, v15
+; VI-NEXT: v_lshrrev_b32_e32 v27, 16, v14
+; VI-NEXT: v_lshrrev_b32_e32 v32, 16, v13
+; VI-NEXT: v_lshrrev_b32_e32 v29, 16, v12
+; VI-NEXT: v_lshrrev_b32_e32 v34, 16, v11
+; VI-NEXT: v_lshrrev_b32_e32 v31, 16, v10
+; VI-NEXT: v_lshrrev_b32_e32 v36, 16, v9
+; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v8
+; VI-NEXT: v_lshrrev_b32_e32 v38, 16, v7
+; VI-NEXT: v_lshrrev_b32_e32 v35, 16, v6
+; VI-NEXT: v_lshrrev_b32_e32 v48, 16, v5
+; VI-NEXT: v_lshrrev_b32_e32 v37, 16, v4
+; VI-NEXT: v_lshrrev_b32_e32 v50, 16, v3
+; VI-NEXT: v_lshrrev_b32_e32 v39, 16, v2
+; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v1
+; VI-NEXT: v_lshrrev_b32_e32 v49, 16, v0
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -30312,7 +30843,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v28, s40
; VI-NEXT: v_mov_b32_e32 v26, s15
; VI-NEXT: v_mov_b32_e32 v24, s14
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; VI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -30372,7 +30903,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s14, s9, 16
; GFX9-NEXT: s_lshr_b32 s57, s8, 16
@@ -30396,8 +30927,38 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s74, s18, 16
; GFX9-NEXT: s_lshr_b32 s56, s17, 16
; GFX9-NEXT: s_lshr_b32 s75, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[12:13], -1
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s12, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
@@ -30431,32 +30992,8 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -30501,7 +31038,7 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v28, s40
; GFX9-NEXT: v_mov_b32_e32 v26, s15
; GFX9-NEXT: v_mov_b32_e32 v24, s14
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -30557,12 +31094,12 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s29, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s28, 16
@@ -30582,9 +31119,38 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[28:29], 1.0
@@ -30618,32 +31184,8 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -30664,9 +31206,9 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s56 :: v_dual_mov_b32 v32, s13
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s47 :: v_dual_mov_b32 v30, s12
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s46 :: v_dual_mov_b32 v28, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s45 :: v_dual_mov_b32 v26, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s43 :: v_dual_mov_b32 v24, s9
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s45 :: v_dual_mov_b32 v26, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s43 :: v_dual_mov_b32 v24, s8
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v51.l
@@ -30701,12 +31243,12 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s8, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s8, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s9, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s29, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s28, 16
@@ -30726,9 +31268,38 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[6:7], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[28:29], 1.0
@@ -30762,32 +31333,8 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v23, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v9, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v7, s22
@@ -30809,8 +31356,8 @@ define inreg <44 x half> @bitcast_v11f64_to_v44f16_scalar(<11 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, s41 :: v_dual_mov_b32 v35, s14
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s15 :: v_dual_mov_b32 v33, s13
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s12 :: v_dual_mov_b32 v29, s11
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s10 :: v_dual_mov_b32 v27, s9
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v28, s9 :: v_dual_mov_b32 v27, s8
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v23, 0xffff, v23
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
@@ -31928,26 +32475,24 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v32, s34, 0
-; SI-NEXT: v_writelane_b32 v32, s35, 1
-; SI-NEXT: v_writelane_b32 v32, s36, 2
-; SI-NEXT: v_writelane_b32 v32, s37, 3
-; SI-NEXT: v_writelane_b32 v32, s38, 4
-; SI-NEXT: v_writelane_b32 v32, s39, 5
-; SI-NEXT: v_writelane_b32 v32, s48, 6
-; SI-NEXT: v_writelane_b32 v32, s49, 7
-; SI-NEXT: v_writelane_b32 v32, s50, 8
-; SI-NEXT: v_writelane_b32 v32, s51, 9
-; SI-NEXT: v_writelane_b32 v32, s52, 10
-; SI-NEXT: v_writelane_b32 v32, s53, 11
-; SI-NEXT: v_writelane_b32 v32, s54, 12
-; SI-NEXT: v_writelane_b32 v32, s55, 13
-; SI-NEXT: v_writelane_b32 v32, s64, 14
-; SI-NEXT: v_writelane_b32 v32, s65, 15
-; SI-NEXT: v_writelane_b32 v32, s66, 16
-; SI-NEXT: v_writelane_b32 v32, s67, 17
-; SI-NEXT: v_writelane_b32 v32, s30, 18
-; SI-NEXT: v_writelane_b32 v32, s31, 19
+; SI-NEXT: v_writelane_b32 v32, s36, 0
+; SI-NEXT: v_writelane_b32 v32, s37, 1
+; SI-NEXT: v_writelane_b32 v32, s38, 2
+; SI-NEXT: v_writelane_b32 v32, s39, 3
+; SI-NEXT: v_writelane_b32 v32, s48, 4
+; SI-NEXT: v_writelane_b32 v32, s49, 5
+; SI-NEXT: v_writelane_b32 v32, s50, 6
+; SI-NEXT: v_writelane_b32 v32, s51, 7
+; SI-NEXT: v_writelane_b32 v32, s52, 8
+; SI-NEXT: v_writelane_b32 v32, s53, 9
+; SI-NEXT: v_writelane_b32 v32, s54, 10
+; SI-NEXT: v_writelane_b32 v32, s55, 11
+; SI-NEXT: v_writelane_b32 v32, s64, 12
+; SI-NEXT: v_writelane_b32 v32, s65, 13
+; SI-NEXT: v_writelane_b32 v32, s66, 14
+; SI-NEXT: v_writelane_b32 v32, s67, 15
+; SI-NEXT: v_writelane_b32 v32, s30, 16
+; SI-NEXT: v_writelane_b32 v32, s31, 17
; SI-NEXT: v_readfirstlane_b32 s6, v7
; SI-NEXT: v_readfirstlane_b32 s8, v6
; SI-NEXT: v_readfirstlane_b32 s10, v5
@@ -31966,10 +32511,10 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: s_lshr_b32 s93, s22, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s95, s20, 16
-; SI-NEXT: s_lshr_b32 s30, s19, 16
-; SI-NEXT: s_lshr_b32 s31, s18, 16
-; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -31980,18 +32525,18 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: s_lshr_b32 s79, s77, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -32048,11 +32593,20 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s57, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -32062,14 +32616,14 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v4, vcc_lo
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -32226,11 +32780,8 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v20, v21, v20
; SI-NEXT: v_lshlrev_b32_e32 v21, 16, v22
; SI-NEXT: v_or_b32_e32 v21, v23, v21
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -32263,27 +32814,25 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 18
-; SI-NEXT: v_readlane_b32 s31, v32, 19
-; SI-NEXT: v_readlane_b32 s67, v32, 17
-; SI-NEXT: v_readlane_b32 s66, v32, 16
-; SI-NEXT: v_readlane_b32 s65, v32, 15
-; SI-NEXT: v_readlane_b32 s64, v32, 14
-; SI-NEXT: v_readlane_b32 s55, v32, 13
-; SI-NEXT: v_readlane_b32 s54, v32, 12
-; SI-NEXT: v_readlane_b32 s53, v32, 11
-; SI-NEXT: v_readlane_b32 s52, v32, 10
-; SI-NEXT: v_readlane_b32 s51, v32, 9
-; SI-NEXT: v_readlane_b32 s50, v32, 8
-; SI-NEXT: v_readlane_b32 s49, v32, 7
-; SI-NEXT: v_readlane_b32 s48, v32, 6
-; SI-NEXT: v_readlane_b32 s39, v32, 5
-; SI-NEXT: v_readlane_b32 s38, v32, 4
-; SI-NEXT: v_readlane_b32 s37, v32, 3
-; SI-NEXT: v_readlane_b32 s36, v32, 2
-; SI-NEXT: v_readlane_b32 s35, v32, 1
-; SI-NEXT: v_readlane_b32 s34, v32, 0
+; SI-NEXT: .LBB55_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 16
+; SI-NEXT: v_readlane_b32 s31, v32, 17
+; SI-NEXT: v_readlane_b32 s67, v32, 15
+; SI-NEXT: v_readlane_b32 s66, v32, 14
+; SI-NEXT: v_readlane_b32 s65, v32, 13
+; SI-NEXT: v_readlane_b32 s64, v32, 12
+; SI-NEXT: v_readlane_b32 s55, v32, 11
+; SI-NEXT: v_readlane_b32 s54, v32, 10
+; SI-NEXT: v_readlane_b32 s53, v32, 9
+; SI-NEXT: v_readlane_b32 s52, v32, 8
+; SI-NEXT: v_readlane_b32 s51, v32, 7
+; SI-NEXT: v_readlane_b32 s50, v32, 6
+; SI-NEXT: v_readlane_b32 s49, v32, 5
+; SI-NEXT: v_readlane_b32 s48, v32, 4
+; SI-NEXT: v_readlane_b32 s39, v32, 3
+; SI-NEXT: v_readlane_b32 s38, v32, 2
+; SI-NEXT: v_readlane_b32 s37, v32, 1
+; SI-NEXT: v_readlane_b32 s36, v32, 0
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -32316,10 +32865,8 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: v_writelane_b32 v32, s67, 17
; VI-NEXT: v_writelane_b32 v32, s68, 18
; VI-NEXT: v_writelane_b32 v32, s69, 19
-; VI-NEXT: v_writelane_b32 v32, s70, 20
-; VI-NEXT: v_writelane_b32 v32, s71, 21
-; VI-NEXT: v_writelane_b32 v32, s30, 22
-; VI-NEXT: v_writelane_b32 v32, s31, 23
+; VI-NEXT: v_writelane_b32 v32, s30, 20
+; VI-NEXT: v_writelane_b32 v32, s31, 21
; VI-NEXT: v_readfirstlane_b32 s6, v7
; VI-NEXT: v_readfirstlane_b32 s8, v6
; VI-NEXT: v_readfirstlane_b32 s10, v5
@@ -32334,14 +32881,14 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: s_lshr_b32 s77, s26, 16
; VI-NEXT: s_lshr_b32 s88, s25, 16
; VI-NEXT: s_lshr_b32 s90, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s68, s19, 16
-; VI-NEXT: s_lshr_b32 s69, s18, 16
-; VI-NEXT: s_lshr_b32 s70, s17, 16
-; VI-NEXT: s_lshr_b32 s71, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s68, s17, 16
+; VI-NEXT: s_lshr_b32 s69, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -32352,31 +32899,31 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: s_lshr_b32 s91, s89, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s70, 16
+; VI-NEXT: s_lshl_b32 s41, s68, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -32420,38 +32967,47 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v21, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s71
-; VI-NEXT: v_mov_b32_e32 v2, s70
+; VI-NEXT: v_mov_b32_e32 v0, s69
+; VI-NEXT: v_mov_b32_e32 v2, s68
; VI-NEXT: v_add_f16_sdwa v0, v0, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v21
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v21
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s69
+; VI-NEXT: v_mov_b32_e32 v2, s35
; VI-NEXT: v_add_f16_sdwa v2, v2, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v21
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s68
+; VI-NEXT: v_mov_b32_e32 v3, s34
; VI-NEXT: v_add_f16_sdwa v3, v3, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v21
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s35
+; VI-NEXT: v_mov_b32_e32 v4, s31
; VI-NEXT: v_add_f16_sdwa v4, v4, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v21
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s34
+; VI-NEXT: v_mov_b32_e32 v5, s30
; VI-NEXT: v_add_f16_sdwa v5, v5, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v21
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s31
+; VI-NEXT: v_mov_b32_e32 v6, vcc_hi
; VI-NEXT: v_add_f16_sdwa v6, v6, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v21
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s30
+; VI-NEXT: v_mov_b32_e32 v7, vcc_lo
; VI-NEXT: v_add_f16_sdwa v7, v7, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v21
; VI-NEXT: v_or_b32_e32 v7, v8, v7
@@ -32511,11 +33067,8 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v22, v22, v21 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s6, v21
; VI-NEXT: v_or_b32_e32 v21, v21, v22
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -32548,11 +33101,9 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 22
-; VI-NEXT: v_readlane_b32 s31, v32, 23
-; VI-NEXT: v_readlane_b32 s71, v32, 21
-; VI-NEXT: v_readlane_b32 s70, v32, 20
+; VI-NEXT: .LBB55_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 20
+; VI-NEXT: v_readlane_b32 s31, v32, 21
; VI-NEXT: v_readlane_b32 s69, v32, 19
; VI-NEXT: v_readlane_b32 s68, v32, 18
; VI-NEXT: v_readlane_b32 s67, v32, 17
@@ -32651,10 +33202,18 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s55, s60, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s56, s56, s59
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s58
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v21, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v21 op_sel_hi:[1,0]
@@ -32678,10 +33237,8 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v19, s55, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v20, s56, v21 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v21, s57, v21 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -32714,7 +33271,7 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -32787,11 +33344,16 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s19, s57, s62
; GFX11-NEXT: s_pack_ll_b32_b16 s20, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -32815,8 +33377,6 @@ define inreg <11 x double> @bitcast_v44f16_to_v11f64_scalar(<44 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v20, 0x200, s20 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v21, 0x200, s21 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -33869,232 +34429,261 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v22, s83, 25
; SI-NEXT: v_writelane_b32 v22, s84, 26
; SI-NEXT: v_writelane_b32 v22, s85, 27
-; SI-NEXT: v_writelane_b32 v22, s30, 28
-; SI-NEXT: v_writelane_b32 v22, s31, 29
-; SI-NEXT: v_readfirstlane_b32 s83, v7
-; SI-NEXT: v_readfirstlane_b32 s85, v6
-; SI-NEXT: v_readfirstlane_b32 s80, v5
-; SI-NEXT: v_readfirstlane_b32 s82, v4
-; SI-NEXT: v_readfirstlane_b32 s69, v3
-; SI-NEXT: v_readfirstlane_b32 s71, v2
-; SI-NEXT: v_readfirstlane_b32 s66, v1
-; SI-NEXT: v_readfirstlane_b32 s68, v0
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s65, s28, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s64, s26, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s55, s24, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s54, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s53, s20, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s52, s18, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
-; SI-NEXT: s_lshr_b32 s51, s16, 16
-; SI-NEXT: s_lshr_b32 s50, s83, 16
-; SI-NEXT: s_lshr_b32 s84, s85, 16
-; SI-NEXT: s_lshr_b32 s49, s80, 16
-; SI-NEXT: s_lshr_b32 s81, s82, 16
-; SI-NEXT: s_lshr_b32 s48, s69, 16
-; SI-NEXT: s_lshr_b32 s70, s71, 16
-; SI-NEXT: s_lshr_b32 s39, s66, 16
-; SI-NEXT: s_lshr_b32 s67, s68, 16
+; SI-NEXT: v_writelane_b32 v22, s86, 28
+; SI-NEXT: v_writelane_b32 v22, s87, 29
+; SI-NEXT: v_writelane_b32 v22, s30, 30
+; SI-NEXT: v_writelane_b32 v22, s31, 31
+; SI-NEXT: v_readfirstlane_b32 s85, v7
+; SI-NEXT: v_readfirstlane_b32 s87, v6
+; SI-NEXT: v_readfirstlane_b32 s82, v5
+; SI-NEXT: v_readfirstlane_b32 s84, v4
+; SI-NEXT: v_readfirstlane_b32 s71, v3
+; SI-NEXT: v_readfirstlane_b32 s81, v2
+; SI-NEXT: v_readfirstlane_b32 s68, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
+; SI-NEXT: s_lshr_b32 s48, s29, 16
+; SI-NEXT: s_lshr_b32 s67, s28, 16
+; SI-NEXT: s_lshr_b32 s39, s27, 16
+; SI-NEXT: s_lshr_b32 s66, s26, 16
+; SI-NEXT: s_lshr_b32 s38, s25, 16
+; SI-NEXT: s_lshr_b32 s65, s24, 16
+; SI-NEXT: s_lshr_b32 s37, s23, 16
+; SI-NEXT: s_lshr_b32 s64, s22, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s55, s20, 16
+; SI-NEXT: s_lshr_b32 s35, s19, 16
+; SI-NEXT: s_lshr_b32 s54, s18, 16
+; SI-NEXT: s_lshr_b32 s34, s17, 16
+; SI-NEXT: s_lshr_b32 s53, s16, 16
+; SI-NEXT: s_lshr_b32 s52, s85, 16
+; SI-NEXT: s_lshr_b32 s86, s87, 16
+; SI-NEXT: s_lshr_b32 s51, s82, 16
+; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s50, s71, 16
+; SI-NEXT: s_lshr_b32 s80, s81, 16
+; SI-NEXT: s_lshr_b32 s49, s68, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s30, 16
+; SI-NEXT: s_lshl_b32 s7, s34, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s58, s51, 16
-; SI-NEXT: s_or_b32 s59, s5, s7
+; SI-NEXT: s_lshl_b32 s62, s53, 16
+; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s31, 16
-; SI-NEXT: s_or_b32 s12, s4, s58
+; SI-NEXT: s_lshl_b32 s7, s35, 16
+; SI-NEXT: s_or_b32 s56, s4, s62
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s52, 16
-; SI-NEXT: s_or_b32 s61, s5, s7
+; SI-NEXT: s_lshl_b32 s72, s54, 16
+; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s34, 16
-; SI-NEXT: s_or_b32 s10, s4, s60
+; SI-NEXT: s_lshl_b32 s7, s36, 16
+; SI-NEXT: s_or_b32 s46, s4, s72
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s53, 16
-; SI-NEXT: s_or_b32 s63, s5, s7
+; SI-NEXT: s_lshl_b32 s74, s55, 16
+; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s35, 16
-; SI-NEXT: s_or_b32 s8, s4, s62
+; SI-NEXT: s_lshl_b32 s7, s37, 16
+; SI-NEXT: s_or_b32 s44, s4, s74
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s72, s54, 16
-; SI-NEXT: s_or_b32 s73, s5, s7
+; SI-NEXT: s_lshl_b32 s76, s64, 16
+; SI-NEXT: s_or_b32 s77, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s36, 16
-; SI-NEXT: s_or_b32 s6, s4, s72
+; SI-NEXT: s_lshl_b32 s7, s38, 16
+; SI-NEXT: s_or_b32 s42, s4, s76
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s56, s55, 16
-; SI-NEXT: s_or_b32 s57, s5, s7
+; SI-NEXT: s_lshl_b32 s78, s65, 16
+; SI-NEXT: s_or_b32 s79, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s37, 16
-; SI-NEXT: s_or_b32 s4, s4, s56
-; SI-NEXT: s_lshl_b32 s46, s64, 16
-; SI-NEXT: s_or_b32 s47, s5, s7
-; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s38, 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[56:57], 16
-; SI-NEXT: s_and_b32 s56, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s65, 16
-; SI-NEXT: s_or_b32 s45, s5, s7
-; SI-NEXT: s_and_b32 s5, s66, 0xffff
; SI-NEXT: s_lshl_b32 s7, s39, 16
-; SI-NEXT: s_or_b32 s56, s56, s46
-; SI-NEXT: s_lshr_b64 s[76:77], s[46:47], 16
-; SI-NEXT: s_and_b32 s46, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s67, 16
-; SI-NEXT: s_or_b32 s43, s5, s7
-; SI-NEXT: s_and_b32 s5, s69, 0xffff
+; SI-NEXT: s_or_b32 s40, s4, s78
+; SI-NEXT: s_and_b32 s4, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s88, s66, 16
+; SI-NEXT: s_or_b32 s89, s5, s7
+; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s48, 16
-; SI-NEXT: s_or_b32 s46, s46, s44
-; SI-NEXT: s_lshr_b64 s[78:79], s[44:45], 16
-; SI-NEXT: s_and_b32 s44, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s40, s70, 16
-; SI-NEXT: s_or_b32 s41, s5, s7
-; SI-NEXT: s_and_b32 s5, s80, 0xffff
+; SI-NEXT: s_or_b32 s14, s4, s88
+; SI-NEXT: s_and_b32 s4, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s90, s67, 16
+; SI-NEXT: s_or_b32 s91, s5, s7
+; SI-NEXT: s_and_b32 s5, s68, 0xffff
; SI-NEXT: s_lshl_b32 s7, s49, 16
-; SI-NEXT: s_or_b32 s44, s44, s42
-; SI-NEXT: s_lshr_b64 s[88:89], s[42:43], 16
-; SI-NEXT: s_and_b32 s42, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s81, 16
-; SI-NEXT: s_or_b32 s15, s5, s7
-; SI-NEXT: s_and_b32 s5, s83, 0xffff
+; SI-NEXT: s_or_b32 s12, s4, s90
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s92, s69, 16
+; SI-NEXT: s_or_b32 s93, s5, s7
+; SI-NEXT: s_and_b32 s5, s71, 0xffff
; SI-NEXT: s_lshl_b32 s7, s50, 16
-; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_lshr_b64 s[90:91], s[40:41], 16
-; SI-NEXT: s_and_b32 s40, s82, 0xffff
+; SI-NEXT: s_or_b32 s10, s4, s92
+; SI-NEXT: s_and_b32 s4, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s94, s80, 16
; SI-NEXT: s_or_b32 s95, s5, s7
-; SI-NEXT: s_lshl_b32 s94, s84, 16
-; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
-; SI-NEXT: s_and_b32 s14, s85, 0xffff
-; SI-NEXT: s_mov_b32 s13, s59
-; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: s_mov_b32 s11, s61
-; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
-; SI-NEXT: s_mov_b32 s9, s63
-; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
-; SI-NEXT: s_mov_b32 s7, s73
-; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
-; SI-NEXT: s_mov_b32 s5, s57
-; SI-NEXT: s_mov_b32 s57, s47
-; SI-NEXT: s_mov_b32 s47, s45
-; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_mov_b32 s43, s41
-; SI-NEXT: s_mov_b32 s41, s15
-; SI-NEXT: s_or_b32 s14, s14, s94
-; SI-NEXT: s_mov_b32 s15, s95
-; SI-NEXT: s_lshr_b64 s[94:95], s[94:95], 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_and_b32 s5, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s51, 16
+; SI-NEXT: s_or_b32 s8, s4, s94
+; SI-NEXT: s_and_b32 s4, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s30, s83, 16
+; SI-NEXT: s_or_b32 s31, s5, s7
+; SI-NEXT: s_and_b32 s5, s85, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s52, 16
+; SI-NEXT: s_or_b32 s6, s4, s30
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 vcc_lo, s86, 16
+; SI-NEXT: s_or_b32 vcc_hi, s5, s7
+; SI-NEXT: s_or_b32 s4, s4, vcc_lo
+; SI-NEXT: s_lshr_b64 s[58:59], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
+; SI-NEXT: s_mov_b32 s57, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[74:75], 16
+; SI-NEXT: s_mov_b32 s47, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[76:77], 16
+; SI-NEXT: s_mov_b32 s45, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[78:79], 16
+; SI-NEXT: s_mov_b32 s43, s77
+; SI-NEXT: s_lshr_b64 s[76:77], s[88:89], 16
+; SI-NEXT: s_mov_b32 s41, s79
+; SI-NEXT: s_lshr_b64 s[78:79], s[90:91], 16
+; SI-NEXT: s_mov_b32 s15, s89
+; SI-NEXT: s_lshr_b64 s[88:89], s[92:93], 16
+; SI-NEXT: s_mov_b32 s13, s91
+; SI-NEXT: s_lshr_b64 s[90:91], s[94:95], 16
+; SI-NEXT: s_mov_b32 s11, s93
+; SI-NEXT: s_lshr_b64 s[92:93], s[30:31], 16
+; SI-NEXT: s_mov_b32 s9, s95
+; SI-NEXT: s_lshr_b64 s[94:95], vcc, 16
+; SI-NEXT: s_mov_b32 s7, s31
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_mov_b32 s5, vcc_hi
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 vcc, s[30:31], exec
+; SI-NEXT: s_cselect_b32 s59, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s59, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: s_add_i32 s87, s87, 3
+; SI-NEXT: s_and_b32 s4, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_add_i32 s85, s85, 3
-; SI-NEXT: s_and_b32 s4, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s83, s83, 3
-; SI-NEXT: s_add_i32 s14, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_and_b32 s5, s85, 0xffff
+; SI-NEXT: s_lshl_b32 s6, s52, 16
+; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: s_and_b32 s6, s84, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s83, 16
; SI-NEXT: s_add_i32 s82, s82, 3
-; SI-NEXT: s_add_i32 s15, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
-; SI-NEXT: s_add_i32 s40, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s49, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s6, s7, s6
+; SI-NEXT: s_and_b32 s7, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s8, s51, 16
+; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_and_b32 s8, s81, 0xffff
+; SI-NEXT: s_lshl_b32 s9, s80, 16
; SI-NEXT: s_add_i32 s71, s71, 3
-; SI-NEXT: s_add_i32 s41, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s71, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
-; SI-NEXT: s_add_i32 s42, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s48, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s8, s9, s8
+; SI-NEXT: s_and_b32 s9, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s10, s50, 16
+; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_and_b32 s10, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s11, s69, 16
; SI-NEXT: s_add_i32 s68, s68, 3
-; SI-NEXT: s_add_i32 s43, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s67, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s66, s66, 3
-; SI-NEXT: s_add_i32 s44, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s39, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s10, s11, s10
+; SI-NEXT: s_and_b32 s11, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s12, s49, 16
; SI-NEXT: s_add_i32 s28, s28, 3
-; SI-NEXT: s_add_i32 s45, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s65, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_and_b32 s12, s28, 0xffff
+; SI-NEXT: s_lshl_b32 s13, s67, 16
; SI-NEXT: s_add_i32 s29, s29, 3
-; SI-NEXT: s_add_i32 s46, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s38, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s12, s13, s12
+; SI-NEXT: s_and_b32 s13, s29, 0xffff
+; SI-NEXT: s_lshl_b32 s14, s48, 16
; SI-NEXT: s_add_i32 s26, s26, 3
-; SI-NEXT: s_add_i32 s47, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_and_b32 s14, s26, 0xffff
+; SI-NEXT: s_lshl_b32 s15, s66, 16
; SI-NEXT: s_add_i32 s27, s27, 3
-; SI-NEXT: s_add_i32 s56, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s37, 16
-; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s14, s15, s14
+; SI-NEXT: s_and_b32 s15, s27, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s39, 16
; SI-NEXT: s_add_i32 s24, s24, 3
-; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: s_or_b32 s15, s26, s15
+; SI-NEXT: s_and_b32 s24, s24, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s65, 16
+; SI-NEXT: s_or_b32 s24, s26, s24
; SI-NEXT: s_add_i32 s25, s25, 3
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s6, s36, 16
+; SI-NEXT: s_add_i32 s40, s24, 0x30000
+; SI-NEXT: s_and_b32 s24, s25, 0xffff
+; SI-NEXT: s_lshl_b32 s25, s38, 16
+; SI-NEXT: s_or_b32 s24, s25, s24
; SI-NEXT: s_add_i32 s22, s22, 3
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_and_b32 s6, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s54, 16
+; SI-NEXT: s_add_i32 s41, s24, 0x30000
+; SI-NEXT: s_and_b32 s22, s22, 0xffff
+; SI-NEXT: s_lshl_b32 s24, s64, 16
+; SI-NEXT: s_or_b32 s22, s24, s22
; SI-NEXT: s_add_i32 s23, s23, 3
-; SI-NEXT: s_or_b32 s6, s7, s6
-; SI-NEXT: s_and_b32 s7, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s35, 16
+; SI-NEXT: s_add_i32 s42, s22, 0x30000
+; SI-NEXT: s_and_b32 s22, s23, 0xffff
+; SI-NEXT: s_lshl_b32 s23, s37, 16
+; SI-NEXT: s_or_b32 s22, s23, s22
; SI-NEXT: s_add_i32 s20, s20, 3
-; SI-NEXT: s_or_b32 s7, s8, s7
-; SI-NEXT: s_and_b32 s8, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s53, 16
+; SI-NEXT: s_add_i32 s43, s22, 0x30000
+; SI-NEXT: s_and_b32 s20, s20, 0xffff
+; SI-NEXT: s_lshl_b32 s22, s55, 16
+; SI-NEXT: s_or_b32 s20, s22, s20
; SI-NEXT: s_add_i32 s21, s21, 3
-; SI-NEXT: s_or_b32 s8, s9, s8
-; SI-NEXT: s_and_b32 s9, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s34, 16
+; SI-NEXT: s_add_i32 s44, s20, 0x30000
+; SI-NEXT: s_and_b32 s20, s21, 0xffff
+; SI-NEXT: s_lshl_b32 s21, s36, 16
+; SI-NEXT: s_or_b32 s20, s21, s20
; SI-NEXT: s_add_i32 s18, s18, 3
-; SI-NEXT: s_or_b32 s9, s10, s9
-; SI-NEXT: s_and_b32 s10, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s52, 16
+; SI-NEXT: s_add_i32 s45, s20, 0x30000
+; SI-NEXT: s_and_b32 s18, s18, 0xffff
+; SI-NEXT: s_lshl_b32 s20, s54, 16
+; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
-; SI-NEXT: s_or_b32 s10, s11, s10
-; SI-NEXT: s_and_b32 s11, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s31, 16
+; SI-NEXT: s_add_i32 s46, s18, 0x30000
+; SI-NEXT: s_and_b32 s18, s19, 0xffff
+; SI-NEXT: s_lshl_b32 s19, s35, 16
+; SI-NEXT: s_or_b32 s18, s19, s18
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: s_or_b32 s11, s12, s11
-; SI-NEXT: s_and_b32 s12, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s51, 16
+; SI-NEXT: s_add_i32 s47, s18, 0x30000
+; SI-NEXT: s_and_b32 s16, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s18, s53, 16
+; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
-; SI-NEXT: s_or_b32 s12, s13, s12
-; SI-NEXT: s_and_b32 s13, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s30, 16
-; SI-NEXT: s_or_b32 s13, s16, s13
+; SI-NEXT: s_add_i32 s56, s16, 0x30000
+; SI-NEXT: s_and_b32 s16, s17, 0xffff
+; SI-NEXT: s_lshl_b32 s17, s34, 16
+; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
@@ -34105,119 +34694,124 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_add_i32 s12, s12, 0x30000
; SI-NEXT: s_add_i32 s13, s13, 0x30000
-; SI-NEXT: s_lshr_b64 s[58:59], s[12:13], 16
-; SI-NEXT: s_lshr_b64 s[60:61], s[10:11], 16
-; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[72:73], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[76:77], s[56:57], 16
-; SI-NEXT: s_lshr_b64 s[78:79], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[88:89], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[92:93], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[94:95], s[14:15], 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s11, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s7, 16
-; SI-NEXT: s_lshr_b32 s36, s5, 16
-; SI-NEXT: s_lshr_b32 s37, s57, 16
-; SI-NEXT: s_lshr_b32 s38, s47, 16
-; SI-NEXT: s_lshr_b32 s39, s45, 16
-; SI-NEXT: s_lshr_b32 s48, s43, 16
-; SI-NEXT: s_lshr_b32 s49, s41, 16
-; SI-NEXT: s_lshr_b32 s50, s15, 16
-; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s58, 16
-; SI-NEXT: s_or_b32 s12, s12, s16
-; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s30, 16
-; SI-NEXT: s_or_b32 s13, s13, s16
-; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s60, 16
-; SI-NEXT: s_or_b32 s10, s10, s16
-; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s31, 16
-; SI-NEXT: s_or_b32 s11, s11, s16
-; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s62, 16
-; SI-NEXT: s_or_b32 s8, s8, s16
-; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s34, 16
-; SI-NEXT: s_or_b32 s9, s9, s16
-; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s72, 16
-; SI-NEXT: s_or_b32 s6, s6, s16
-; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s35, 16
-; SI-NEXT: s_or_b32 s7, s7, s16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s74, 16
-; SI-NEXT: s_or_b32 s4, s4, s16
-; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s36, 16
-; SI-NEXT: s_or_b32 s5, s5, s16
+; SI-NEXT: s_add_i32 s14, s14, 0x30000
+; SI-NEXT: s_add_i32 s15, s15, 0x30000
+; SI-NEXT: s_add_i32 s57, s16, 0x30000
+; SI-NEXT: s_lshr_b64 s[58:59], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[60:61], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[62:63], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[88:89], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[6:7], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
+; SI-NEXT: s_lshr_b32 s34, s57, 16
+; SI-NEXT: s_lshr_b32 s35, s47, 16
+; SI-NEXT: s_lshr_b32 s36, s45, 16
+; SI-NEXT: s_lshr_b32 s37, s43, 16
+; SI-NEXT: s_lshr_b32 s38, s41, 16
+; SI-NEXT: s_lshr_b32 s39, s15, 16
+; SI-NEXT: s_lshr_b32 s48, s13, 16
+; SI-NEXT: s_lshr_b32 s49, s11, 16
+; SI-NEXT: s_lshr_b32 s50, s9, 16
+; SI-NEXT: s_lshr_b32 s51, s7, 16
+; SI-NEXT: s_lshr_b32 s52, s5, 16
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s16, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s76, 16
+; SI-NEXT: s_lshl_b32 s17, s58, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s37, 16
+; SI-NEXT: s_lshl_b32 s18, s34, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s78, 16
+; SI-NEXT: s_lshl_b32 s19, s60, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s38, 16
+; SI-NEXT: s_lshl_b32 s20, s35, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s44, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s88, 16
+; SI-NEXT: s_lshl_b32 s21, s62, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s39, 16
+; SI-NEXT: s_lshl_b32 s22, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s90, 16
+; SI-NEXT: s_lshl_b32 s23, s72, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s48, 16
+; SI-NEXT: s_lshl_b32 s24, s37, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s92, 16
+; SI-NEXT: s_lshl_b32 s25, s74, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s49, 16
+; SI-NEXT: s_lshl_b32 s26, s38, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s94, 16
+; SI-NEXT: s_lshl_b32 s26, s76, 16
; SI-NEXT: s_or_b32 s14, s14, s26
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s50, 16
+; SI-NEXT: s_lshl_b32 s26, s39, 16
; SI-NEXT: s_or_b32 s15, s15, s26
-; SI-NEXT: v_readlane_b32 s30, v22, 28
-; SI-NEXT: v_mov_b32_e32 v0, s12
-; SI-NEXT: v_mov_b32_e32 v1, s13
-; SI-NEXT: v_mov_b32_e32 v2, s10
-; SI-NEXT: v_mov_b32_e32 v3, s11
-; SI-NEXT: v_mov_b32_e32 v4, s8
-; SI-NEXT: v_mov_b32_e32 v5, s9
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s4
-; SI-NEXT: v_mov_b32_e32 v9, s5
-; SI-NEXT: v_mov_b32_e32 v10, s16
-; SI-NEXT: v_mov_b32_e32 v11, s17
-; SI-NEXT: v_mov_b32_e32 v12, s18
-; SI-NEXT: v_mov_b32_e32 v13, s19
-; SI-NEXT: v_mov_b32_e32 v14, s20
-; SI-NEXT: v_mov_b32_e32 v15, s21
-; SI-NEXT: v_mov_b32_e32 v16, s22
-; SI-NEXT: v_mov_b32_e32 v17, s23
-; SI-NEXT: v_mov_b32_e32 v18, s24
-; SI-NEXT: v_mov_b32_e32 v19, s25
-; SI-NEXT: v_mov_b32_e32 v20, s14
-; SI-NEXT: v_mov_b32_e32 v21, s15
-; SI-NEXT: v_readlane_b32 s31, v22, 29
+; SI-NEXT: s_and_b32 s12, s12, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s78, 16
+; SI-NEXT: s_or_b32 s12, s12, s26
+; SI-NEXT: s_and_b32 s13, s13, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s48, 16
+; SI-NEXT: s_or_b32 s13, s13, s26
+; SI-NEXT: s_and_b32 s10, s10, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s88, 16
+; SI-NEXT: s_or_b32 s10, s10, s26
+; SI-NEXT: s_and_b32 s11, s11, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s49, 16
+; SI-NEXT: s_or_b32 s11, s11, s26
+; SI-NEXT: s_and_b32 s8, s8, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s90, 16
+; SI-NEXT: s_or_b32 s8, s8, s26
+; SI-NEXT: s_and_b32 s9, s9, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s50, 16
+; SI-NEXT: s_or_b32 s9, s9, s26
+; SI-NEXT: s_and_b32 s6, s6, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s92, 16
+; SI-NEXT: s_or_b32 s6, s6, s26
+; SI-NEXT: s_and_b32 s7, s7, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s51, 16
+; SI-NEXT: s_or_b32 s7, s7, s26
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s94, 16
+; SI-NEXT: s_or_b32 s4, s4, s26
+; SI-NEXT: s_and_b32 s5, s5, 0xffff
+; SI-NEXT: s_lshl_b32 s26, s52, 16
+; SI-NEXT: s_or_b32 s5, s5, s26
+; SI-NEXT: v_readlane_b32 s30, v22, 30
+; SI-NEXT: v_mov_b32_e32 v0, s16
+; SI-NEXT: v_mov_b32_e32 v1, s17
+; SI-NEXT: v_mov_b32_e32 v2, s18
+; SI-NEXT: v_mov_b32_e32 v3, s19
+; SI-NEXT: v_mov_b32_e32 v4, s20
+; SI-NEXT: v_mov_b32_e32 v5, s21
+; SI-NEXT: v_mov_b32_e32 v6, s22
+; SI-NEXT: v_mov_b32_e32 v7, s23
+; SI-NEXT: v_mov_b32_e32 v8, s24
+; SI-NEXT: v_mov_b32_e32 v9, s25
+; SI-NEXT: v_mov_b32_e32 v10, s14
+; SI-NEXT: v_mov_b32_e32 v11, s15
+; SI-NEXT: v_mov_b32_e32 v12, s12
+; SI-NEXT: v_mov_b32_e32 v13, s13
+; SI-NEXT: v_mov_b32_e32 v14, s10
+; SI-NEXT: v_mov_b32_e32 v15, s11
+; SI-NEXT: v_mov_b32_e32 v16, s8
+; SI-NEXT: v_mov_b32_e32 v17, s9
+; SI-NEXT: v_mov_b32_e32 v18, s6
+; SI-NEXT: v_mov_b32_e32 v19, s7
+; SI-NEXT: v_mov_b32_e32 v20, s4
+; SI-NEXT: v_mov_b32_e32 v21, s5
+; SI-NEXT: v_readlane_b32 s31, v22, 31
+; SI-NEXT: v_readlane_b32 s87, v22, 29
+; SI-NEXT: v_readlane_b32 s86, v22, 28
; SI-NEXT: v_readlane_b32 s85, v22, 27
; SI-NEXT: v_readlane_b32 s84, v22, 26
; SI-NEXT: v_readlane_b32 s83, v22, 25
@@ -34251,30 +34845,6 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v44i16_to_v44f16_scalar:
; VI: ; %bb.0:
@@ -34311,10 +34881,18 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s60, s59, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s75, s75, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -34359,7 +34937,7 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s9, s9, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -34449,8 +35027,6 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v20, s8
; VI-NEXT: v_mov_b32_e32 v21, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v44i16_to_v44f16_scalar:
; GFX9: ; %bb.0:
@@ -34487,10 +35063,18 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s60, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v8
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s75, s59
; GFX9-NEXT: v_pk_add_u16 v21, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s74, s58
@@ -34557,10 +35141,8 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v23, 16, v20
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v21, s75
; GFX9-NEXT: v_mov_b32_e32 v20, s74
; GFX9-NEXT: v_mov_b32_e32 v19, s73
@@ -34605,7 +35187,7 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v49, s8
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -34684,11 +35266,16 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s58, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s62, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s62, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s62, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s61, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s56, s60, s56
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s59, s47
@@ -34756,8 +35343,6 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v20
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v21
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s61 :: v_dual_mov_b32 v20, s60
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s59 :: v_dual_mov_b32 v18, s58
@@ -34840,11 +35425,16 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s58, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s62, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s62, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s62, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s61, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s56, s60, s56
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s59, s47
@@ -34912,8 +35502,6 @@ define inreg <44 x half> @bitcast_v44i16_to_v44f16_scalar(<44 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 16, v18
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s61 :: v_dual_mov_b32 v18, s60
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s59 :: v_dual_mov_b32 v20, s58
@@ -35766,10 +36354,18 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s61, s63, 16
; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s75
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s74
@@ -35964,10 +36560,8 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[26:27], v[16:17], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[18:19], 16
; SI-NEXT: v_lshr_b64 v[22:23], v[20:21], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v42, s58
; SI-NEXT: v_mov_b32_e32 v44, s56
; SI-NEXT: v_mov_b32_e32 v45, s46
@@ -36018,7 +36612,7 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v26, s47
; SI-NEXT: v_mov_b32_e32 v24, s45
; SI-NEXT: v_mov_b32_e32 v22, s43
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v50
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -36139,10 +36733,18 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s75, s74, 16
; VI-NEXT: v_readfirstlane_b32 s4, v8
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v22, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v22
; VI-NEXT: v_add_f16_e32 v51, s43, v22
@@ -36188,10 +36790,8 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v23, s47, v22
; VI-NEXT: v_add_f16_e32 v21, s44, v22
; VI-NEXT: v_add_f16_e32 v22, s45, v22
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v22, s45
; VI-NEXT: v_mov_b32_e32 v21, s44
; VI-NEXT: v_mov_b32_e32 v23, s47
@@ -36236,7 +36836,7 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v51, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; VI-NEXT: v_lshlrev_b32_e32 v50, 16, v50
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
@@ -36318,10 +36918,18 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s60, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v8
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s75, s59
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v21, s4, v0 op_sel_hi:[1,0]
@@ -36389,10 +36997,8 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v23, 16, v20
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v21, s75
; GFX9-NEXT: v_mov_b32_e32 v20, s74
; GFX9-NEXT: v_mov_b32_e32 v19, s73
@@ -36437,7 +37043,7 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v49, s8
; GFX9-NEXT: v_mov_b32_e32 v50, s7
; GFX9-NEXT: v_mov_b32_e32 v51, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -36516,11 +37122,16 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s58, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s62, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s62, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s62, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s61, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s56, s60, s56
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s59, s47
@@ -36588,8 +37199,6 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v20
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v21
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s61 :: v_dual_mov_b32 v20, s60
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s59 :: v_dual_mov_b32 v18, s58
@@ -36672,11 +37281,16 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s58, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s62, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s62, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s62, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s62, s62, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s62, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s62, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s61, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s56, s60, s56
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s59, s47
@@ -36744,8 +37358,6 @@ define inreg <44 x i16> @bitcast_v44f16_to_v44i16_scalar(<44 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 16, v18
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, s61 :: v_dual_mov_b32 v18, s60
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s59 :: v_dual_mov_b32 v20, s58
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
index 332c550eaaccf..d6e590ca5df5e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll
@@ -188,10 +188,18 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -216,7 +224,7 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -242,8 +250,6 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v24i32_to_v24f32_scalar:
; VI: ; %bb.0:
@@ -260,10 +266,18 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -288,7 +302,7 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -314,8 +328,6 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v24i32_to_v24f32_scalar:
; GFX9: ; %bb.0:
@@ -332,10 +344,18 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -360,7 +380,7 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -386,8 +406,6 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v24i32_to_v24f32_scalar:
; GFX11: ; %bb.0:
@@ -401,11 +419,16 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -430,7 +453,7 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -445,8 +468,6 @@ define inreg <24 x float> @bitcast_v24i32_to_v24f32_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -664,10 +685,18 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
; SI-NEXT: v_add_f32_e64 v22, s58, 1.0
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -692,10 +721,8 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -728,7 +755,7 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -791,10 +818,18 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
; VI-NEXT: v_add_f32_e64 v22, s58, 1.0
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -819,10 +854,8 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -855,7 +888,7 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -918,10 +951,18 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX9-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -946,10 +987,8 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -982,7 +1021,7 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -1046,11 +1085,16 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX11-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -1076,8 +1120,6 @@ define inreg <24 x i32> @bitcast_v24f32_to_v24i32_scalar(<24 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1312,10 +1354,18 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1340,7 +1390,7 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1366,8 +1416,6 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v24i32_to_v12i64_scalar:
; VI: ; %bb.0:
@@ -1384,10 +1432,18 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1412,7 +1468,7 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1438,8 +1494,6 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v24i32_to_v12i64_scalar:
; GFX9: ; %bb.0:
@@ -1456,10 +1510,18 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1484,7 +1546,7 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1510,8 +1572,6 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v24i32_to_v12i64_scalar:
; GFX11: ; %bb.0:
@@ -1525,11 +1585,16 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1554,7 +1619,7 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1569,8 +1634,6 @@ define inreg <12 x i64> @bitcast_v24i32_to_v12i64_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1776,10 +1839,18 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1804,7 +1875,7 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1830,8 +1901,6 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v12i64_to_v24i32_scalar:
; VI: ; %bb.0:
@@ -1848,10 +1917,18 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -1876,7 +1953,7 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1902,8 +1979,6 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v12i64_to_v24i32_scalar:
; GFX9: ; %bb.0:
@@ -1920,10 +1995,18 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -1948,7 +2031,7 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1974,8 +2057,6 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v12i64_to_v24i32_scalar:
; GFX11: ; %bb.0:
@@ -1989,11 +2070,16 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -2018,7 +2104,7 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2033,8 +2119,6 @@ define inreg <24 x i32> @bitcast_v12i64_to_v24i32_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2234,10 +2318,18 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2262,7 +2354,7 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2288,8 +2380,6 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v24i32_to_v12f64_scalar:
; VI: ; %bb.0:
@@ -2306,10 +2396,18 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2334,7 +2432,7 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2360,8 +2458,6 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v24i32_to_v12f64_scalar:
; GFX9: ; %bb.0:
@@ -2378,10 +2474,18 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2406,7 +2510,7 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2432,8 +2536,6 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v24i32_to_v12f64_scalar:
; GFX11: ; %bb.0:
@@ -2447,11 +2549,16 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -2476,7 +2583,7 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2491,8 +2598,6 @@ define inreg <12 x double> @bitcast_v24i32_to_v12f64_scalar(<24 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2674,10 +2779,18 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -2690,10 +2803,8 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2726,7 +2837,7 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2789,10 +2900,18 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -2805,10 +2924,8 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -2841,7 +2958,7 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2904,10 +3021,18 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -2920,10 +3045,8 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -2956,7 +3079,7 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -3020,11 +3143,16 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -3038,8 +3166,6 @@ define inreg <24 x i32> @bitcast_v12f64_to_v24i32_scalar(<12 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -3774,10 +3900,8 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v24, s34, 0
-; SI-NEXT: v_writelane_b32 v24, s35, 1
-; SI-NEXT: v_writelane_b32 v24, s30, 2
-; SI-NEXT: v_writelane_b32 v24, s31, 3
+; SI-NEXT: v_writelane_b32 v24, s30, 0
+; SI-NEXT: v_writelane_b32 v24, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -3790,7 +3914,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s88, s5, 16
; SI-NEXT: s_lshr_b32 s89, s7, 16
@@ -3800,10 +3924,10 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -3816,8 +3940,40 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -3858,38 +4014,38 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[62:63], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s15, s76, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s16, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s35, 16
+; SI-NEXT: s_lshl_b32 s17, s31, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s74, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_and_b32 s18, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s34, 16
+; SI-NEXT: s_lshl_b32 s19, s30, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_lshl_b32 s19, s72, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s19, s20, s19
; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s31, 16
+; SI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s22, 0xffff
; SI-NEXT: s_lshl_b32 s22, s62, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s30, 16
+; SI-NEXT: s_lshl_b32 s23, vcc_lo, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s24, 0xffff
; SI-NEXT: s_lshl_b32 s24, s60, 16
@@ -3939,7 +4095,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s14, s88, 16
; SI-NEXT: s_or_b32 s7, s7, s29
; SI-NEXT: s_or_b32 s5, s5, s14
-; SI-NEXT: v_readlane_b32 s30, v24, 2
+; SI-NEXT: v_readlane_b32 s30, v24, 0
; SI-NEXT: v_mov_b32_e32 v0, s15
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v2, s17
@@ -3964,40 +4120,12 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v21, s7
; SI-NEXT: v_mov_b32_e32 v22, s4
; SI-NEXT: v_mov_b32_e32 v23, s5
-; SI-NEXT: v_readlane_b32 s31, v24, 3
-; SI-NEXT: v_readlane_b32 s35, v24, 1
-; SI-NEXT: v_readlane_b32 s34, v24, 0
+; SI-NEXT: v_readlane_b32 s31, v24, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v24i32_to_v48i16_scalar:
; VI: ; %bb.0:
@@ -4014,7 +4142,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -4040,8 +4168,40 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -4090,7 +4250,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -4188,32 +4348,6 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v24i32_to_v48i16_scalar:
; GFX9: ; %bb.0:
@@ -4230,7 +4364,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -4256,8 +4390,40 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -4306,7 +4472,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s79
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s78
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s77
@@ -4356,32 +4522,6 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v24i32_to_v48i16_scalar:
; GFX11: ; %bb.0:
@@ -4395,7 +4535,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s74, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s10, s4, 16
; GFX11-NEXT: s_lshr_b32 s11, s5, 16
@@ -4421,9 +4561,40 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s74, -1
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -4472,7 +4643,7 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s73
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s72
@@ -4511,32 +4682,6 @@ define inreg <48 x i16> @bitcast_v24i32_to_v48i16_scalar(<24 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5629,10 +5774,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v24, s67, 17
; SI-NEXT: v_writelane_b32 v24, s68, 18
; SI-NEXT: v_writelane_b32 v24, s69, 19
-; SI-NEXT: v_writelane_b32 v24, s70, 20
-; SI-NEXT: v_writelane_b32 v24, s71, 21
-; SI-NEXT: v_writelane_b32 v24, s30, 22
-; SI-NEXT: v_writelane_b32 v24, s31, 23
+; SI-NEXT: v_writelane_b32 v24, s30, 20
+; SI-NEXT: v_writelane_b32 v24, s31, 21
; SI-NEXT: v_readfirstlane_b32 s7, v9
; SI-NEXT: v_readfirstlane_b32 s9, v8
; SI-NEXT: v_readfirstlane_b32 s11, v7
@@ -5649,14 +5792,14 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s89, s26, 16
; SI-NEXT: s_lshr_b32 s92, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -5669,31 +5812,31 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -5743,46 +5886,55 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -5865,8 +6017,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s59, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v24, 22
+; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v24, 20
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -5891,9 +6043,7 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v21, s57
; SI-NEXT: v_mov_b32_e32 v22, s58
; SI-NEXT: v_mov_b32_e32 v23, s59
-; SI-NEXT: v_readlane_b32 s31, v24, 23
-; SI-NEXT: v_readlane_b32 s71, v24, 21
-; SI-NEXT: v_readlane_b32 s70, v24, 20
+; SI-NEXT: v_readlane_b32 s31, v24, 21
; SI-NEXT: v_readlane_b32 s69, v24, 19
; SI-NEXT: v_readlane_b32 s68, v24, 18
; SI-NEXT: v_readlane_b32 s67, v24, 17
@@ -5919,9 +6069,6 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v48i16_to_v24i32_scalar:
; VI: ; %bb.0:
@@ -5953,10 +6100,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v24, s71, 21
; VI-NEXT: v_writelane_b32 v24, s80, 22
; VI-NEXT: v_writelane_b32 v24, s81, 23
-; VI-NEXT: v_writelane_b32 v24, s82, 24
-; VI-NEXT: v_writelane_b32 v24, s83, 25
-; VI-NEXT: v_writelane_b32 v24, s30, 26
-; VI-NEXT: v_writelane_b32 v24, s31, 27
+; VI-NEXT: v_writelane_b32 v24, s30, 24
+; VI-NEXT: v_writelane_b32 v24, s31, 25
; VI-NEXT: v_readfirstlane_b32 s7, v9
; VI-NEXT: v_readfirstlane_b32 s9, v8
; VI-NEXT: v_readfirstlane_b32 s11, v7
@@ -5966,21 +6111,21 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s77, v3
; VI-NEXT: v_readfirstlane_b32 s88, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s75, s28, 16
; VI-NEXT: s_lshr_b32 s78, s27, 16
; VI-NEXT: s_lshr_b32 s89, s26, 16
-; VI-NEXT: s_lshr_b32 s30, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s25, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -5990,40 +6135,40 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: s_lshr_b32 s79, s88, 16
; VI-NEXT: s_lshr_b32 s90, s91, 16
-; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -6037,8 +6182,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -6067,56 +6212,65 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -6138,10 +6292,10 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
@@ -6189,8 +6343,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s59, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v24, 26
+; VI-NEXT: .LBB15_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v24, 24
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -6215,9 +6369,7 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v21, s57
; VI-NEXT: v_mov_b32_e32 v22, s58
; VI-NEXT: v_mov_b32_e32 v23, s59
-; VI-NEXT: v_readlane_b32 s31, v24, 27
-; VI-NEXT: v_readlane_b32 s83, v24, 25
-; VI-NEXT: v_readlane_b32 s82, v24, 24
+; VI-NEXT: v_readlane_b32 s31, v24, 25
; VI-NEXT: v_readlane_b32 s81, v24, 23
; VI-NEXT: v_readlane_b32 s80, v24, 22
; VI-NEXT: v_readlane_b32 s71, v24, 21
@@ -6247,9 +6399,6 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v48i16_to_v24i32_scalar:
; GFX9: ; %bb.0:
@@ -6329,10 +6478,18 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -6357,10 +6514,8 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v22, s58, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -6393,7 +6548,7 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -6472,11 +6627,16 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -6502,8 +6662,6 @@ define inreg <24 x i32> @bitcast_v48i16_to_v24i32_scalar(<48 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v22, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v23, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7221,10 +7379,8 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v24, s34, 0
-; SI-NEXT: v_writelane_b32 v24, s35, 1
-; SI-NEXT: v_writelane_b32 v24, s30, 2
-; SI-NEXT: v_writelane_b32 v24, s31, 3
+; SI-NEXT: v_writelane_b32 v24, s30, 0
+; SI-NEXT: v_writelane_b32 v24, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -7237,7 +7393,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s88, s5, 16
; SI-NEXT: s_lshr_b32 s89, s7, 16
@@ -7247,10 +7403,10 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -7263,8 +7419,40 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -7305,38 +7493,38 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[62:63], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s15, s76, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s16, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s35, 16
+; SI-NEXT: s_lshl_b32 s17, s31, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s74, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_and_b32 s18, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s34, 16
+; SI-NEXT: s_lshl_b32 s19, s30, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_lshl_b32 s19, s72, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s19, s20, s19
; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s31, 16
+; SI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s22, 0xffff
; SI-NEXT: s_lshl_b32 s22, s62, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s30, 16
+; SI-NEXT: s_lshl_b32 s23, vcc_lo, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s24, 0xffff
; SI-NEXT: s_lshl_b32 s24, s60, 16
@@ -7386,7 +7574,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: s_lshl_b32 s14, s88, 16
; SI-NEXT: s_or_b32 s7, s7, s29
; SI-NEXT: s_or_b32 s5, s5, s14
-; SI-NEXT: v_readlane_b32 s30, v24, 2
+; SI-NEXT: v_readlane_b32 s30, v24, 0
; SI-NEXT: v_mov_b32_e32 v0, s15
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v2, s17
@@ -7411,40 +7599,12 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v21, s7
; SI-NEXT: v_mov_b32_e32 v22, s4
; SI-NEXT: v_mov_b32_e32 v23, s5
-; SI-NEXT: v_readlane_b32 s31, v24, 3
-; SI-NEXT: v_readlane_b32 s35, v24, 1
-; SI-NEXT: v_readlane_b32 s34, v24, 0
+; SI-NEXT: v_readlane_b32 s31, v24, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v24i32_to_v48f16_scalar:
; VI: ; %bb.0:
@@ -7461,7 +7621,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -7487,8 +7647,40 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -7537,7 +7729,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -7635,32 +7827,6 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v24i32_to_v48f16_scalar:
; GFX9: ; %bb.0:
@@ -7677,7 +7843,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -7703,8 +7869,40 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -7753,7 +7951,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s79
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s78
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s77
@@ -7803,32 +8001,6 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v24i32_to_v48f16_scalar:
; GFX11: ; %bb.0:
@@ -7842,7 +8014,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s74, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s10, s4, 16
; GFX11-NEXT: s_lshr_b32 s11, s5, 16
@@ -7868,9 +8040,40 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s74, -1
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -7919,7 +8122,7 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s73
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s72
@@ -7958,32 +8161,6 @@ define inreg <48 x half> @bitcast_v24i32_to_v48f16_scalar(<24 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9188,10 +9365,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s67, 17
; SI-NEXT: v_writelane_b32 v32, s68, 18
; SI-NEXT: v_writelane_b32 v32, s69, 19
-; SI-NEXT: v_writelane_b32 v32, s70, 20
-; SI-NEXT: v_writelane_b32 v32, s71, 21
-; SI-NEXT: v_writelane_b32 v32, s30, 22
-; SI-NEXT: v_writelane_b32 v32, s31, 23
+; SI-NEXT: v_writelane_b32 v32, s30, 20
+; SI-NEXT: v_writelane_b32 v32, s31, 21
; SI-NEXT: v_readfirstlane_b32 s6, v9
; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_readfirstlane_b32 s10, v7
@@ -9208,14 +9383,14 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -9228,31 +9403,31 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s89, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -9302,11 +9477,20 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s71
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s69
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -9316,7 +9500,7 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -9326,20 +9510,20 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s34
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s30
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -9349,7 +9533,7 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v6, vcc_hi
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -9358,7 +9542,7 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v7, vcc_lo
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
@@ -9496,11 +9680,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v22, v23, v22
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; SI-NEXT: v_or_b32_e32 v23, v25, v23
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -9533,11 +9714,9 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 22
-; SI-NEXT: v_readlane_b32 s31, v32, 23
-; SI-NEXT: v_readlane_b32 s71, v32, 21
-; SI-NEXT: v_readlane_b32 s70, v32, 20
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 20
+; SI-NEXT: v_readlane_b32 s31, v32, 21
; SI-NEXT: v_readlane_b32 s69, v32, 19
; SI-NEXT: v_readlane_b32 s68, v32, 18
; SI-NEXT: v_readlane_b32 s67, v32, 17
@@ -9594,10 +9773,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s71, 21
; VI-NEXT: v_writelane_b32 v32, s80, 22
; VI-NEXT: v_writelane_b32 v32, s81, 23
-; VI-NEXT: v_writelane_b32 v32, s82, 24
-; VI-NEXT: v_writelane_b32 v32, s83, 25
-; VI-NEXT: v_writelane_b32 v32, s30, 26
-; VI-NEXT: v_writelane_b32 v32, s31, 27
+; VI-NEXT: v_writelane_b32 v32, s30, 24
+; VI-NEXT: v_writelane_b32 v32, s31, 25
; VI-NEXT: v_readfirstlane_b32 s6, v9
; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s10, v7
@@ -9607,21 +9784,21 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s76, v3
; VI-NEXT: v_readfirstlane_b32 s79, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s74, s28, 16
; VI-NEXT: s_lshr_b32 s77, s27, 16
; VI-NEXT: s_lshr_b32 s88, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
-; VI-NEXT: s_lshr_b32 s31, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -9630,38 +9807,38 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s75, s73, 16
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -9678,11 +9855,11 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s51, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -9708,42 +9885,51 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v23, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s83
-; VI-NEXT: v_mov_b32_e32 v2, s82
+; VI-NEXT: v_mov_b32_e32 v0, s81
+; VI-NEXT: v_mov_b32_e32 v2, s80
; VI-NEXT: v_add_f16_sdwa v0, v0, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v23
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v23
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_mov_b32_e32 v2, s71
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v23
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v3, s70
; VI-NEXT: v_add_f16_sdwa v3, v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v23
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s71
+; VI-NEXT: v_mov_b32_e32 v4, s69
; VI-NEXT: v_add_f16_sdwa v4, v4, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v23
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s70
+; VI-NEXT: v_mov_b32_e32 v5, s68
; VI-NEXT: v_add_f16_sdwa v5, v5, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v23
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s69
+; VI-NEXT: v_mov_b32_e32 v6, s35
; VI-NEXT: v_add_f16_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v23
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s68
+; VI-NEXT: v_mov_b32_e32 v7, s34
; VI-NEXT: v_add_f16_sdwa v7, v7, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v23
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s31
+; VI-NEXT: v_mov_b32_e32 v8, vcc_hi
; VI-NEXT: v_add_f16_sdwa v8, v8, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v23
; VI-NEXT: v_or_b32_e32 v8, v9, v8
@@ -9767,11 +9953,11 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v13, v13, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v23
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s35
+; VI-NEXT: v_mov_b32_e32 v14, s31
; VI-NEXT: v_add_f16_sdwa v14, v14, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s34, v23
+; VI-NEXT: v_add_f16_e32 v15, s30, v23
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s30
+; VI-NEXT: v_mov_b32_e32 v15, vcc_lo
; VI-NEXT: v_add_f16_sdwa v15, v15, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s91, v23
; VI-NEXT: v_or_b32_e32 v15, v16, v15
@@ -9807,11 +9993,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v24, v24, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s6, v23
; VI-NEXT: v_or_b32_e32 v23, v23, v24
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -9844,11 +10027,9 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 26
-; VI-NEXT: v_readlane_b32 s31, v32, 27
-; VI-NEXT: v_readlane_b32 s83, v32, 25
-; VI-NEXT: v_readlane_b32 s82, v32, 24
+; VI-NEXT: .LBB19_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 24
+; VI-NEXT: v_readlane_b32 s31, v32, 25
; VI-NEXT: v_readlane_b32 s81, v32, 23
; VI-NEXT: v_readlane_b32 s80, v32, 22
; VI-NEXT: v_readlane_b32 s71, v32, 21
@@ -9957,10 +10138,18 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v23, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v23 op_sel_hi:[1,0]
@@ -9986,10 +10175,8 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v21, s57, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v22, s58, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v23, s59, v23 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10022,7 +10209,7 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -10101,11 +10288,16 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -10131,8 +10323,6 @@ define inreg <24 x i32> @bitcast_v48f16_to_v24i32_scalar(<48 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v22, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v23, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -10368,10 +10558,18 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
; SI-NEXT: v_add_f32_e64 v22, s58, 1.0
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -10396,10 +10594,8 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10432,7 +10628,7 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10495,10 +10691,18 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
; VI-NEXT: v_add_f32_e64 v22, s58, 1.0
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -10523,10 +10727,8 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -10559,7 +10761,7 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -10622,10 +10824,18 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX9-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -10650,10 +10860,8 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10686,7 +10894,7 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -10750,11 +10958,16 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX11-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -10780,8 +10993,6 @@ define inreg <12 x i64> @bitcast_v24f32_to_v12i64_scalar(<24 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -11022,10 +11233,18 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -11050,7 +11269,7 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -11076,8 +11295,6 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v12i64_to_v24f32_scalar:
; VI: ; %bb.0:
@@ -11094,10 +11311,18 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -11122,7 +11347,7 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11148,8 +11373,6 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v12i64_to_v24f32_scalar:
; GFX9: ; %bb.0:
@@ -11166,10 +11389,18 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -11194,7 +11425,7 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11220,8 +11451,6 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v12i64_to_v24f32_scalar:
; GFX11: ; %bb.0:
@@ -11235,11 +11464,16 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -11264,7 +11498,7 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11279,8 +11513,6 @@ define inreg <24 x float> @bitcast_v12i64_to_v24f32_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11498,10 +11730,18 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
; SI-NEXT: v_add_f32_e64 v22, s58, 1.0
; SI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -11526,10 +11766,8 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11562,7 +11800,7 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11625,10 +11863,18 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
; VI-NEXT: v_add_f32_e64 v22, s58, 1.0
; VI-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -11653,10 +11899,8 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -11689,7 +11933,7 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11752,10 +11996,18 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX9-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX9-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -11780,10 +12032,8 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -11816,7 +12066,7 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -11880,11 +12130,16 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
; GFX11-NEXT: v_add_f32_e64 v22, s58, 1.0
; GFX11-NEXT: v_add_f32_e64 v21, s57, 1.0
@@ -11910,8 +12165,6 @@ define inreg <12 x double> @bitcast_v24f32_to_v12f64_scalar(<24 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -12128,10 +12381,18 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -12144,10 +12405,8 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -12180,7 +12439,7 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12243,10 +12502,18 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -12259,10 +12526,8 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -12295,7 +12560,7 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12358,10 +12623,18 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -12374,10 +12647,8 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -12410,7 +12681,7 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -12474,11 +12745,16 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
@@ -12492,8 +12768,6 @@ define inreg <24 x float> @bitcast_v12f64_to_v24f32_scalar(<12 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -13204,10 +13478,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v40, s34, 0
-; SI-NEXT: v_writelane_b32 v40, s35, 1
-; SI-NEXT: v_writelane_b32 v40, s30, 2
-; SI-NEXT: v_writelane_b32 v40, s31, 3
+; SI-NEXT: v_writelane_b32 v40, s30, 0
+; SI-NEXT: v_writelane_b32 v40, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -13220,12 +13492,12 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s35, s5, 16
-; SI-NEXT: s_lshr_b32 s34, s7, 16
-; SI-NEXT: s_lshr_b32 s31, s9, 16
-; SI-NEXT: s_lshr_b32 s30, s11, 16
+; SI-NEXT: s_lshr_b32 s31, s5, 16
+; SI-NEXT: s_lshr_b32 s30, s7, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s9, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s11, 16
; SI-NEXT: s_lshr_b32 s95, s13, 16
; SI-NEXT: s_lshr_b32 s94, s29, 16
; SI-NEXT: s_lshr_b32 s93, s27, 16
@@ -13246,8 +13518,40 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v23, s5, 1.0
; SI-NEXT: v_add_f32_e64 v22, s4, 1.0
; SI-NEXT: v_add_f32_e64 v21, s7, 1.0
@@ -13296,34 +13600,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13356,10 +13634,10 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v50, s93
; SI-NEXT: v_mov_b32_e32 v49, s94
; SI-NEXT: v_mov_b32_e32 v48, s95
-; SI-NEXT: v_mov_b32_e32 v39, s30
-; SI-NEXT: v_mov_b32_e32 v38, s31
-; SI-NEXT: v_mov_b32_e32 v37, s34
-; SI-NEXT: v_mov_b32_e32 v36, s35
+; SI-NEXT: v_mov_b32_e32 v39, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v38, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v37, s30
+; SI-NEXT: v_mov_b32_e32 v36, s31
; SI-NEXT: v_mov_b32_e32 v24, s14
; SI-NEXT: v_mov_b32_e32 v25, s40
; SI-NEXT: v_mov_b32_e32 v26, s42
@@ -13372,7 +13650,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v33, s72
; SI-NEXT: v_mov_b32_e32 v34, s74
; SI-NEXT: v_mov_b32_e32 v35, s76
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
@@ -13433,7 +13711,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v36
-; SI-NEXT: v_readlane_b32 s30, v40, 2
+; SI-NEXT: v_readlane_b32 s30, v40, 0
; SI-NEXT: v_or_b32_e32 v1, v1, v35
; SI-NEXT: v_or_b32_e32 v3, v3, v34
; SI-NEXT: v_or_b32_e32 v5, v5, v33
@@ -13446,9 +13724,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v19, v19, v26
; SI-NEXT: v_or_b32_e32 v21, v21, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_readlane_b32 s31, v40, 3
-; SI-NEXT: v_readlane_b32 s35, v40, 1
-; SI-NEXT: v_readlane_b32 s34, v40, 0
+; SI-NEXT: v_readlane_b32 s31, v40, 1
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -13470,7 +13746,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -13496,8 +13772,40 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v23, s6, 1.0
; VI-NEXT: v_add_f32_e64 v22, s7, 1.0
; VI-NEXT: v_add_f32_e64 v21, s8, 1.0
@@ -13546,34 +13854,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13622,7 +13904,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s42
; VI-NEXT: v_mov_b32_e32 v25, s41
; VI-NEXT: v_mov_b32_e32 v24, s40
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
; VI-NEXT: v_lshlrev_b32_e32 v54, 16, v54
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
@@ -13688,7 +13970,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -13714,8 +13996,40 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v23, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v22, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v21, s8, 1.0
@@ -13764,34 +14078,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13840,7 +14128,7 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -13902,11 +14190,11 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s8, 16
@@ -13929,9 +14217,40 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s72, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v23, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v22, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v21, s6, 1.0
@@ -13980,34 +14299,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -14031,8 +14324,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s11 :: v_dual_mov_b32 v24, s10
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v55.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v54.l
@@ -14071,11 +14364,11 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s8, 16
@@ -14098,9 +14391,40 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s72, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v19, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v20, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v21, s6, 1.0
@@ -14149,34 +14473,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v2, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v7, s17
@@ -14200,8 +14498,8 @@ define inreg <48 x i16> @bitcast_v24f32_to_v48i16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s11 :: v_dual_mov_b32 v24, s10
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v1
@@ -15343,10 +15641,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v24, s67, 17
; SI-NEXT: v_writelane_b32 v24, s68, 18
; SI-NEXT: v_writelane_b32 v24, s69, 19
-; SI-NEXT: v_writelane_b32 v24, s70, 20
-; SI-NEXT: v_writelane_b32 v24, s71, 21
-; SI-NEXT: v_writelane_b32 v24, s30, 22
-; SI-NEXT: v_writelane_b32 v24, s31, 23
+; SI-NEXT: v_writelane_b32 v24, s30, 20
+; SI-NEXT: v_writelane_b32 v24, s31, 21
; SI-NEXT: v_readfirstlane_b32 s7, v9
; SI-NEXT: v_readfirstlane_b32 s9, v8
; SI-NEXT: v_readfirstlane_b32 s11, v7
@@ -15363,14 +15659,14 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s89, s26, 16
; SI-NEXT: s_lshr_b32 s92, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -15383,31 +15679,31 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -15457,46 +15753,55 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -15579,8 +15884,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s59, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v24, 22
+; SI-NEXT: .LBB31_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v24, 20
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -15605,9 +15910,7 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v21, s57
; SI-NEXT: v_mov_b32_e32 v22, s58
; SI-NEXT: v_mov_b32_e32 v23, s59
-; SI-NEXT: v_readlane_b32 s31, v24, 23
-; SI-NEXT: v_readlane_b32 s71, v24, 21
-; SI-NEXT: v_readlane_b32 s70, v24, 20
+; SI-NEXT: v_readlane_b32 s31, v24, 21
; SI-NEXT: v_readlane_b32 s69, v24, 19
; SI-NEXT: v_readlane_b32 s68, v24, 18
; SI-NEXT: v_readlane_b32 s67, v24, 17
@@ -15633,9 +15936,6 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v48i16_to_v24f32_scalar:
; VI: ; %bb.0:
@@ -15667,10 +15967,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v24, s71, 21
; VI-NEXT: v_writelane_b32 v24, s80, 22
; VI-NEXT: v_writelane_b32 v24, s81, 23
-; VI-NEXT: v_writelane_b32 v24, s82, 24
-; VI-NEXT: v_writelane_b32 v24, s83, 25
-; VI-NEXT: v_writelane_b32 v24, s30, 26
-; VI-NEXT: v_writelane_b32 v24, s31, 27
+; VI-NEXT: v_writelane_b32 v24, s30, 24
+; VI-NEXT: v_writelane_b32 v24, s31, 25
; VI-NEXT: v_readfirstlane_b32 s7, v9
; VI-NEXT: v_readfirstlane_b32 s9, v8
; VI-NEXT: v_readfirstlane_b32 s11, v7
@@ -15680,21 +15978,21 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s77, v3
; VI-NEXT: v_readfirstlane_b32 s88, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s75, s28, 16
; VI-NEXT: s_lshr_b32 s78, s27, 16
; VI-NEXT: s_lshr_b32 s89, s26, 16
-; VI-NEXT: s_lshr_b32 s30, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s25, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -15704,40 +16002,40 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: s_lshr_b32 s79, s88, 16
; VI-NEXT: s_lshr_b32 s90, s91, 16
-; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -15751,8 +16049,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -15781,56 +16079,65 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -15852,10 +16159,10 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
@@ -15903,8 +16210,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s59, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v24, 26
+; VI-NEXT: .LBB31_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v24, 24
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -15929,9 +16236,7 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v21, s57
; VI-NEXT: v_mov_b32_e32 v22, s58
; VI-NEXT: v_mov_b32_e32 v23, s59
-; VI-NEXT: v_readlane_b32 s31, v24, 27
-; VI-NEXT: v_readlane_b32 s83, v24, 25
-; VI-NEXT: v_readlane_b32 s82, v24, 24
+; VI-NEXT: v_readlane_b32 s31, v24, 25
; VI-NEXT: v_readlane_b32 s81, v24, 23
; VI-NEXT: v_readlane_b32 s80, v24, 22
; VI-NEXT: v_readlane_b32 s71, v24, 21
@@ -15961,9 +16266,6 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v48i16_to_v24f32_scalar:
; GFX9: ; %bb.0:
@@ -16043,10 +16345,18 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -16071,10 +16381,8 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v22, s58, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -16107,7 +16415,7 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -16186,11 +16494,16 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -16216,8 +16529,6 @@ define inreg <24 x float> @bitcast_v48i16_to_v24f32_scalar(<48 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v22, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v23, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -16911,10 +17222,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v40, s34, 0
-; SI-NEXT: v_writelane_b32 v40, s35, 1
-; SI-NEXT: v_writelane_b32 v40, s30, 2
-; SI-NEXT: v_writelane_b32 v40, s31, 3
+; SI-NEXT: v_writelane_b32 v40, s30, 0
+; SI-NEXT: v_writelane_b32 v40, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -16927,12 +17236,12 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s35, s5, 16
-; SI-NEXT: s_lshr_b32 s34, s7, 16
-; SI-NEXT: s_lshr_b32 s31, s9, 16
-; SI-NEXT: s_lshr_b32 s30, s11, 16
+; SI-NEXT: s_lshr_b32 s31, s5, 16
+; SI-NEXT: s_lshr_b32 s30, s7, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s9, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s11, 16
; SI-NEXT: s_lshr_b32 s95, s13, 16
; SI-NEXT: s_lshr_b32 s94, s29, 16
; SI-NEXT: s_lshr_b32 s93, s27, 16
@@ -16953,8 +17262,40 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v23, s5, 1.0
; SI-NEXT: v_add_f32_e64 v22, s4, 1.0
; SI-NEXT: v_add_f32_e64 v21, s7, 1.0
@@ -17003,34 +17344,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -17063,10 +17378,10 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v50, s93
; SI-NEXT: v_mov_b32_e32 v49, s94
; SI-NEXT: v_mov_b32_e32 v48, s95
-; SI-NEXT: v_mov_b32_e32 v39, s30
-; SI-NEXT: v_mov_b32_e32 v38, s31
-; SI-NEXT: v_mov_b32_e32 v37, s34
-; SI-NEXT: v_mov_b32_e32 v36, s35
+; SI-NEXT: v_mov_b32_e32 v39, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v38, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v37, s30
+; SI-NEXT: v_mov_b32_e32 v36, s31
; SI-NEXT: v_mov_b32_e32 v24, s14
; SI-NEXT: v_mov_b32_e32 v25, s40
; SI-NEXT: v_mov_b32_e32 v26, s42
@@ -17079,7 +17394,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v33, s72
; SI-NEXT: v_mov_b32_e32 v34, s74
; SI-NEXT: v_mov_b32_e32 v35, s76
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v34, 16, v34
@@ -17140,7 +17455,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v36
-; SI-NEXT: v_readlane_b32 s30, v40, 2
+; SI-NEXT: v_readlane_b32 s30, v40, 0
; SI-NEXT: v_or_b32_e32 v1, v1, v35
; SI-NEXT: v_or_b32_e32 v3, v3, v34
; SI-NEXT: v_or_b32_e32 v5, v5, v33
@@ -17153,9 +17468,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v19, v19, v26
; SI-NEXT: v_or_b32_e32 v21, v21, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_readlane_b32 s31, v40, 3
-; SI-NEXT: v_readlane_b32 s35, v40, 1
-; SI-NEXT: v_readlane_b32 s34, v40, 0
+; SI-NEXT: v_readlane_b32 s31, v40, 1
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -17177,7 +17490,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -17203,8 +17516,40 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v23, s6, 1.0
; VI-NEXT: v_add_f32_e64 v22, s7, 1.0
; VI-NEXT: v_add_f32_e64 v21, s8, 1.0
@@ -17253,34 +17598,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -17329,7 +17648,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s42
; VI-NEXT: v_mov_b32_e32 v25, s41
; VI-NEXT: v_mov_b32_e32 v24, s40
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
; VI-NEXT: v_lshlrev_b32_e32 v54, 16, v54
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
@@ -17395,7 +17714,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -17421,8 +17740,40 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v23, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v22, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v21, s8, 1.0
@@ -17471,34 +17822,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17547,7 +17872,7 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s42
; GFX9-NEXT: v_mov_b32_e32 v25, s41
; GFX9-NEXT: v_mov_b32_e32 v24, s40
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -17609,11 +17934,11 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s8, 16
@@ -17636,9 +17961,40 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s72, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v23, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v22, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v21, s6, 1.0
@@ -17687,34 +18043,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -17738,8 +18068,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s11 :: v_dual_mov_b32 v24, s10
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v55.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v54.l
@@ -17778,11 +18108,11 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s8, 16
@@ -17805,9 +18135,40 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s72, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v19, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v20, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v21, s6, 1.0
@@ -17856,34 +18217,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v2, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v7, s17
@@ -17907,8 +18242,8 @@ define inreg <48 x half> @bitcast_v24f32_to_v48f16_scalar(<24 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s12 :: v_dual_mov_b32 v24, s11
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s11 :: v_dual_mov_b32 v24, s10
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v1
@@ -19162,10 +19497,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s67, 17
; SI-NEXT: v_writelane_b32 v32, s68, 18
; SI-NEXT: v_writelane_b32 v32, s69, 19
-; SI-NEXT: v_writelane_b32 v32, s70, 20
-; SI-NEXT: v_writelane_b32 v32, s71, 21
-; SI-NEXT: v_writelane_b32 v32, s30, 22
-; SI-NEXT: v_writelane_b32 v32, s31, 23
+; SI-NEXT: v_writelane_b32 v32, s30, 20
+; SI-NEXT: v_writelane_b32 v32, s31, 21
; SI-NEXT: v_readfirstlane_b32 s6, v9
; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_readfirstlane_b32 s10, v7
@@ -19182,14 +19515,14 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -19202,31 +19535,31 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s91, s89, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -19276,11 +19609,20 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s71
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s69
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -19290,7 +19632,7 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -19300,20 +19642,20 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s34
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s30
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -19323,7 +19665,7 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v6, vcc_hi
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -19332,7 +19674,7 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v7, vcc_lo
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
@@ -19470,11 +19812,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v22, v23, v22
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; SI-NEXT: v_or_b32_e32 v23, v25, v23
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -19507,11 +19846,9 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 22
-; SI-NEXT: v_readlane_b32 s31, v32, 23
-; SI-NEXT: v_readlane_b32 s71, v32, 21
-; SI-NEXT: v_readlane_b32 s70, v32, 20
+; SI-NEXT: .LBB35_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 20
+; SI-NEXT: v_readlane_b32 s31, v32, 21
; SI-NEXT: v_readlane_b32 s69, v32, 19
; SI-NEXT: v_readlane_b32 s68, v32, 18
; SI-NEXT: v_readlane_b32 s67, v32, 17
@@ -19568,10 +19905,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: v_writelane_b32 v32, s71, 21
; VI-NEXT: v_writelane_b32 v32, s80, 22
; VI-NEXT: v_writelane_b32 v32, s81, 23
-; VI-NEXT: v_writelane_b32 v32, s82, 24
-; VI-NEXT: v_writelane_b32 v32, s83, 25
-; VI-NEXT: v_writelane_b32 v32, s30, 26
-; VI-NEXT: v_writelane_b32 v32, s31, 27
+; VI-NEXT: v_writelane_b32 v32, s30, 24
+; VI-NEXT: v_writelane_b32 v32, s31, 25
; VI-NEXT: v_readfirstlane_b32 s6, v9
; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s10, v7
@@ -19581,21 +19916,21 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s76, v3
; VI-NEXT: v_readfirstlane_b32 s79, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s74, s28, 16
; VI-NEXT: s_lshr_b32 s77, s27, 16
; VI-NEXT: s_lshr_b32 s88, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
-; VI-NEXT: s_lshr_b32 s31, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -19604,38 +19939,38 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s75, s73, 16
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -19652,11 +19987,11 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s51, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -19682,42 +20017,51 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v23, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s83
-; VI-NEXT: v_mov_b32_e32 v2, s82
+; VI-NEXT: v_mov_b32_e32 v0, s81
+; VI-NEXT: v_mov_b32_e32 v2, s80
; VI-NEXT: v_add_f16_sdwa v0, v0, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v23
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v23
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_mov_b32_e32 v2, s71
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v23
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v3, s70
; VI-NEXT: v_add_f16_sdwa v3, v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v23
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s71
+; VI-NEXT: v_mov_b32_e32 v4, s69
; VI-NEXT: v_add_f16_sdwa v4, v4, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v23
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s70
+; VI-NEXT: v_mov_b32_e32 v5, s68
; VI-NEXT: v_add_f16_sdwa v5, v5, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v23
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s69
+; VI-NEXT: v_mov_b32_e32 v6, s35
; VI-NEXT: v_add_f16_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v23
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s68
+; VI-NEXT: v_mov_b32_e32 v7, s34
; VI-NEXT: v_add_f16_sdwa v7, v7, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v23
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s31
+; VI-NEXT: v_mov_b32_e32 v8, vcc_hi
; VI-NEXT: v_add_f16_sdwa v8, v8, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v23
; VI-NEXT: v_or_b32_e32 v8, v9, v8
@@ -19741,11 +20085,11 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v13, v13, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v23
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s35
+; VI-NEXT: v_mov_b32_e32 v14, s31
; VI-NEXT: v_add_f16_sdwa v14, v14, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s34, v23
+; VI-NEXT: v_add_f16_e32 v15, s30, v23
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s30
+; VI-NEXT: v_mov_b32_e32 v15, vcc_lo
; VI-NEXT: v_add_f16_sdwa v15, v15, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s91, v23
; VI-NEXT: v_or_b32_e32 v15, v16, v15
@@ -19781,11 +20125,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v24, v24, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s6, v23
; VI-NEXT: v_or_b32_e32 v23, v23, v24
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -19818,11 +20159,9 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 26
-; VI-NEXT: v_readlane_b32 s31, v32, 27
-; VI-NEXT: v_readlane_b32 s83, v32, 25
-; VI-NEXT: v_readlane_b32 s82, v32, 24
+; VI-NEXT: .LBB35_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 24
+; VI-NEXT: v_readlane_b32 s31, v32, 25
; VI-NEXT: v_readlane_b32 s81, v32, 23
; VI-NEXT: v_readlane_b32 s80, v32, 22
; VI-NEXT: v_readlane_b32 s71, v32, 21
@@ -19931,10 +20270,18 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v23, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v23 op_sel_hi:[1,0]
@@ -19960,10 +20307,8 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v21, s57, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v22, s58, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v23, s59, v23 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -19996,7 +20341,7 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -20075,11 +20420,16 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -20105,8 +20455,6 @@ define inreg <24 x float> @bitcast_v48f16_to_v24f32_scalar(<48 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v22, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v23, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -20330,10 +20678,18 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s14, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s15, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -20358,7 +20714,7 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -20384,8 +20740,6 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s7
; SI-NEXT: v_mov_b32_e32 v23, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v12i64_to_v12f64_scalar:
; VI: ; %bb.0:
@@ -20402,10 +20756,18 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -20430,7 +20792,7 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -20456,8 +20818,6 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v12i64_to_v12f64_scalar:
; GFX9: ; %bb.0:
@@ -20474,10 +20834,18 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -20502,7 +20870,7 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20528,8 +20896,6 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v12i64_to_v12f64_scalar:
; GFX11: ; %bb.0:
@@ -20543,11 +20909,16 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s10, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s10, s10, exec_lo
+; GFX11-NEXT: s_cselect_b32 s10, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s10, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -20572,7 +20943,7 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -20586,8 +20957,6 @@ define inreg <12 x double> @bitcast_v12i64_to_v12f64_scalar(<12 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -20769,10 +21138,18 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -20785,10 +21162,8 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; SI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -20821,7 +21196,7 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -20884,10 +21259,18 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -20900,10 +21283,8 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; VI-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -20936,7 +21317,7 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -20999,10 +21380,18 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -21015,10 +21404,8 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[18:19], s[54:55], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -21051,7 +21438,7 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -21115,11 +21502,16 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -21133,8 +21525,6 @@ define inreg <12 x i64> @bitcast_v12f64_to_v12i64_scalar(<12 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -21881,10 +22271,8 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v24, s34, 0
-; SI-NEXT: v_writelane_b32 v24, s35, 1
-; SI-NEXT: v_writelane_b32 v24, s30, 2
-; SI-NEXT: v_writelane_b32 v24, s31, 3
+; SI-NEXT: v_writelane_b32 v24, s30, 0
+; SI-NEXT: v_writelane_b32 v24, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -21897,7 +22285,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s88, s5, 16
; SI-NEXT: s_lshr_b32 s89, s7, 16
@@ -21907,10 +22295,10 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -21923,8 +22311,40 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -21957,10 +22377,10 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -21973,30 +22393,30 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s15, s76, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s16, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s35, 16
+; SI-NEXT: s_lshl_b32 s17, s31, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s74, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_and_b32 s18, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s34, 16
+; SI-NEXT: s_lshl_b32 s19, s30, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_lshl_b32 s19, s72, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s19, s20, s19
; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s31, 16
+; SI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s22, 0xffff
; SI-NEXT: s_lshl_b32 s22, s62, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s30, 16
+; SI-NEXT: s_lshl_b32 s23, vcc_lo, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s24, 0xffff
; SI-NEXT: s_lshl_b32 s24, s60, 16
@@ -22046,7 +22466,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s14, s88, 16
; SI-NEXT: s_or_b32 s7, s7, s29
; SI-NEXT: s_or_b32 s5, s5, s14
-; SI-NEXT: v_readlane_b32 s30, v24, 2
+; SI-NEXT: v_readlane_b32 s30, v24, 0
; SI-NEXT: v_mov_b32_e32 v0, s15
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v2, s17
@@ -22071,40 +22491,12 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v21, s7
; SI-NEXT: v_mov_b32_e32 v22, s4
; SI-NEXT: v_mov_b32_e32 v23, s5
-; SI-NEXT: v_readlane_b32 s31, v24, 3
-; SI-NEXT: v_readlane_b32 s35, v24, 1
-; SI-NEXT: v_readlane_b32 s34, v24, 0
+; SI-NEXT: v_readlane_b32 s31, v24, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v12i64_to_v48i16_scalar:
; VI: ; %bb.0:
@@ -22121,7 +22513,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -22147,8 +22539,40 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -22197,7 +22621,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -22295,32 +22719,6 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v12i64_to_v48i16_scalar:
; GFX9: ; %bb.0:
@@ -22337,7 +22735,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -22363,8 +22761,40 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -22413,7 +22843,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s79
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s78
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s77
@@ -22463,32 +22893,6 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v12i64_to_v48i16_scalar:
; GFX11: ; %bb.0:
@@ -22502,7 +22906,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s74, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s10, s4, 16
; GFX11-NEXT: s_lshr_b32 s11, s5, 16
@@ -22528,9 +22932,40 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s74, -1
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -22579,7 +23014,7 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s73
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s72
@@ -22618,32 +23053,6 @@ define inreg <48 x i16> @bitcast_v12i64_to_v48i16_scalar(<12 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -23736,10 +24145,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v24, s67, 17
; SI-NEXT: v_writelane_b32 v24, s68, 18
; SI-NEXT: v_writelane_b32 v24, s69, 19
-; SI-NEXT: v_writelane_b32 v24, s70, 20
-; SI-NEXT: v_writelane_b32 v24, s71, 21
-; SI-NEXT: v_writelane_b32 v24, s30, 22
-; SI-NEXT: v_writelane_b32 v24, s31, 23
+; SI-NEXT: v_writelane_b32 v24, s30, 20
+; SI-NEXT: v_writelane_b32 v24, s31, 21
; SI-NEXT: v_readfirstlane_b32 s7, v9
; SI-NEXT: v_readfirstlane_b32 s9, v8
; SI-NEXT: v_readfirstlane_b32 s11, v7
@@ -23756,14 +24163,14 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s89, s26, 16
; SI-NEXT: s_lshr_b32 s92, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -23776,31 +24183,31 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -23850,46 +24257,55 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -23972,8 +24388,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s59, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v24, 22
+; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v24, 20
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -23998,9 +24414,7 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v21, s57
; SI-NEXT: v_mov_b32_e32 v22, s58
; SI-NEXT: v_mov_b32_e32 v23, s59
-; SI-NEXT: v_readlane_b32 s31, v24, 23
-; SI-NEXT: v_readlane_b32 s71, v24, 21
-; SI-NEXT: v_readlane_b32 s70, v24, 20
+; SI-NEXT: v_readlane_b32 s31, v24, 21
; SI-NEXT: v_readlane_b32 s69, v24, 19
; SI-NEXT: v_readlane_b32 s68, v24, 18
; SI-NEXT: v_readlane_b32 s67, v24, 17
@@ -24026,9 +24440,6 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v48i16_to_v12i64_scalar:
; VI: ; %bb.0:
@@ -24060,10 +24471,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v24, s71, 21
; VI-NEXT: v_writelane_b32 v24, s80, 22
; VI-NEXT: v_writelane_b32 v24, s81, 23
-; VI-NEXT: v_writelane_b32 v24, s82, 24
-; VI-NEXT: v_writelane_b32 v24, s83, 25
-; VI-NEXT: v_writelane_b32 v24, s30, 26
-; VI-NEXT: v_writelane_b32 v24, s31, 27
+; VI-NEXT: v_writelane_b32 v24, s30, 24
+; VI-NEXT: v_writelane_b32 v24, s31, 25
; VI-NEXT: v_readfirstlane_b32 s7, v9
; VI-NEXT: v_readfirstlane_b32 s9, v8
; VI-NEXT: v_readfirstlane_b32 s11, v7
@@ -24073,21 +24482,21 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s77, v3
; VI-NEXT: v_readfirstlane_b32 s88, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s75, s28, 16
; VI-NEXT: s_lshr_b32 s78, s27, 16
; VI-NEXT: s_lshr_b32 s89, s26, 16
-; VI-NEXT: s_lshr_b32 s30, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s25, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -24097,40 +24506,40 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: s_lshr_b32 s79, s88, 16
; VI-NEXT: s_lshr_b32 s90, s91, 16
-; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -24144,8 +24553,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -24174,56 +24583,65 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -24245,10 +24663,10 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
@@ -24296,8 +24714,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s59, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v24, 26
+; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v24, 24
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -24322,9 +24740,7 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v21, s57
; VI-NEXT: v_mov_b32_e32 v22, s58
; VI-NEXT: v_mov_b32_e32 v23, s59
-; VI-NEXT: v_readlane_b32 s31, v24, 27
-; VI-NEXT: v_readlane_b32 s83, v24, 25
-; VI-NEXT: v_readlane_b32 s82, v24, 24
+; VI-NEXT: v_readlane_b32 s31, v24, 25
; VI-NEXT: v_readlane_b32 s81, v24, 23
; VI-NEXT: v_readlane_b32 s80, v24, 22
; VI-NEXT: v_readlane_b32 s71, v24, 21
@@ -24354,9 +24770,6 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v48i16_to_v12i64_scalar:
; GFX9: ; %bb.0:
@@ -24436,10 +24849,18 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -24464,10 +24885,8 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v22, s58, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -24500,7 +24919,7 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -24579,11 +24998,16 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -24609,8 +25033,6 @@ define inreg <12 x i64> @bitcast_v48i16_to_v12i64_scalar(<48 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v22, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v23, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -25340,10 +25762,8 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v24, s34, 0
-; SI-NEXT: v_writelane_b32 v24, s35, 1
-; SI-NEXT: v_writelane_b32 v24, s30, 2
-; SI-NEXT: v_writelane_b32 v24, s31, 3
+; SI-NEXT: v_writelane_b32 v24, s30, 0
+; SI-NEXT: v_writelane_b32 v24, s31, 1
; SI-NEXT: v_readfirstlane_b32 s12, v10
; SI-NEXT: v_readfirstlane_b32 s5, v9
; SI-NEXT: v_readfirstlane_b32 s4, v8
@@ -25356,7 +25776,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s13, v1
; SI-NEXT: s_cmp_lg_u32 s12, 0
; SI-NEXT: v_readfirstlane_b32 s12, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s88, s5, 16
; SI-NEXT: s_lshr_b32 s89, s7, 16
@@ -25366,10 +25786,10 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -25382,8 +25802,40 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -25416,10 +25868,10 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s29, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
; SI-NEXT: s_lshr_b32 s95, s25, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s21, 16
-; SI-NEXT: s_lshr_b32 s34, s19, 16
-; SI-NEXT: s_lshr_b32 s35, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s21, 16
+; SI-NEXT: s_lshr_b32 s30, s19, 16
+; SI-NEXT: s_lshr_b32 s31, s17, 16
; SI-NEXT: s_lshr_b64 s[14:15], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[40:41], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[8:9], 16
@@ -25432,30 +25884,30 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s15, s76, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s15, s16, s15
; SI-NEXT: s_and_b32 s16, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s35, 16
+; SI-NEXT: s_lshl_b32 s17, s31, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_lshl_b32 s17, s74, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_and_b32 s18, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s34, 16
+; SI-NEXT: s_lshl_b32 s19, s30, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_lshl_b32 s19, s72, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s19, s20, s19
; SI-NEXT: s_and_b32 s20, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s31, 16
+; SI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s22, 0xffff
; SI-NEXT: s_lshl_b32 s22, s62, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s30, 16
+; SI-NEXT: s_lshl_b32 s23, vcc_lo, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s24, 0xffff
; SI-NEXT: s_lshl_b32 s24, s60, 16
@@ -25505,7 +25957,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: s_lshl_b32 s14, s88, 16
; SI-NEXT: s_or_b32 s7, s7, s29
; SI-NEXT: s_or_b32 s5, s5, s14
-; SI-NEXT: v_readlane_b32 s30, v24, 2
+; SI-NEXT: v_readlane_b32 s30, v24, 0
; SI-NEXT: v_mov_b32_e32 v0, s15
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v2, s17
@@ -25530,40 +25982,12 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v21, s7
; SI-NEXT: v_mov_b32_e32 v22, s4
; SI-NEXT: v_mov_b32_e32 v23, s5
-; SI-NEXT: v_readlane_b32 s31, v24, 3
-; SI-NEXT: v_readlane_b32 s35, v24, 1
-; SI-NEXT: v_readlane_b32 s34, v24, 0
+; SI-NEXT: v_readlane_b32 s31, v24, 1
; SI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v12i64_to_v48f16_scalar:
; VI: ; %bb.0:
@@ -25580,7 +26004,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s14, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s15, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s6, 16
; VI-NEXT: s_lshr_b32 s41, s7, 16
@@ -25606,8 +26030,40 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -25656,7 +26112,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s77, s18, 16
; VI-NEXT: s_lshr_b32 s78, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -25754,32 +26210,6 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v22, s7
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v12i64_to_v48f16_scalar:
; GFX9: ; %bb.0:
@@ -25796,7 +26226,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s14, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s15, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s6, 16
; GFX9-NEXT: s_lshr_b32 s41, s7, 16
@@ -25822,8 +26252,40 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -25872,7 +26334,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s77, s18, 16
; GFX9-NEXT: s_lshr_b32 s78, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s79
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s78
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s77
@@ -25922,32 +26384,6 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v22, s7
; GFX9-NEXT: v_mov_b32_e32 v23, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v12i64_to_v48f16_scalar:
; GFX11: ; %bb.0:
@@ -25961,7 +26397,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-NEXT: s_cmp_lg_u32 s10, 0
; GFX11-NEXT: s_mov_b32 s74, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s10, s4, 16
; GFX11-NEXT: s_lshr_b32 s11, s5, 16
@@ -25987,9 +26423,40 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s74, -1
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -26038,7 +26505,7 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s63, s2, 16
; GFX11-NEXT: s_lshr_b32 s72, s1, 16
; GFX11-NEXT: s_lshr_b32 s73, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s73
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s72
@@ -26077,32 +26544,6 @@ define inreg <48 x half> @bitcast_v12i64_to_v48f16_scalar(<12 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v21, s6
; GFX11-NEXT: v_dual_mov_b32 v22, s5 :: v_dual_mov_b32 v23, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -27307,10 +27748,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s67, 17
; SI-NEXT: v_writelane_b32 v32, s68, 18
; SI-NEXT: v_writelane_b32 v32, s69, 19
-; SI-NEXT: v_writelane_b32 v32, s70, 20
-; SI-NEXT: v_writelane_b32 v32, s71, 21
-; SI-NEXT: v_writelane_b32 v32, s30, 22
-; SI-NEXT: v_writelane_b32 v32, s31, 23
+; SI-NEXT: v_writelane_b32 v32, s30, 20
+; SI-NEXT: v_writelane_b32 v32, s31, 21
; SI-NEXT: v_readfirstlane_b32 s6, v9
; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_readfirstlane_b32 s10, v7
@@ -27327,14 +27766,14 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -27347,31 +27786,31 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s89, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -27421,11 +27860,20 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s71
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s69
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -27435,7 +27883,7 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -27445,20 +27893,20 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s34
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s30
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -27468,7 +27916,7 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v6, vcc_hi
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -27477,7 +27925,7 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v7, vcc_lo
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
@@ -27615,11 +28063,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v22, v23, v22
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; SI-NEXT: v_or_b32_e32 v23, v25, v23
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -27652,11 +28097,9 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 22
-; SI-NEXT: v_readlane_b32 s31, v32, 23
-; SI-NEXT: v_readlane_b32 s71, v32, 21
-; SI-NEXT: v_readlane_b32 s70, v32, 20
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 20
+; SI-NEXT: v_readlane_b32 s31, v32, 21
; SI-NEXT: v_readlane_b32 s69, v32, 19
; SI-NEXT: v_readlane_b32 s68, v32, 18
; SI-NEXT: v_readlane_b32 s67, v32, 17
@@ -27713,10 +28156,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s71, 21
; VI-NEXT: v_writelane_b32 v32, s80, 22
; VI-NEXT: v_writelane_b32 v32, s81, 23
-; VI-NEXT: v_writelane_b32 v32, s82, 24
-; VI-NEXT: v_writelane_b32 v32, s83, 25
-; VI-NEXT: v_writelane_b32 v32, s30, 26
-; VI-NEXT: v_writelane_b32 v32, s31, 27
+; VI-NEXT: v_writelane_b32 v32, s30, 24
+; VI-NEXT: v_writelane_b32 v32, s31, 25
; VI-NEXT: v_readfirstlane_b32 s6, v9
; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s10, v7
@@ -27726,21 +28167,21 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s76, v3
; VI-NEXT: v_readfirstlane_b32 s79, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s74, s28, 16
; VI-NEXT: s_lshr_b32 s77, s27, 16
; VI-NEXT: s_lshr_b32 s88, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
-; VI-NEXT: s_lshr_b32 s31, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -27749,38 +28190,38 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s75, s73, 16
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -27797,11 +28238,11 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s51, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -27827,42 +28268,51 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v23, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s83
-; VI-NEXT: v_mov_b32_e32 v2, s82
+; VI-NEXT: v_mov_b32_e32 v0, s81
+; VI-NEXT: v_mov_b32_e32 v2, s80
; VI-NEXT: v_add_f16_sdwa v0, v0, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v23
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v23
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_mov_b32_e32 v2, s71
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v23
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v3, s70
; VI-NEXT: v_add_f16_sdwa v3, v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v23
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s71
+; VI-NEXT: v_mov_b32_e32 v4, s69
; VI-NEXT: v_add_f16_sdwa v4, v4, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v23
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s70
+; VI-NEXT: v_mov_b32_e32 v5, s68
; VI-NEXT: v_add_f16_sdwa v5, v5, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v23
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s69
+; VI-NEXT: v_mov_b32_e32 v6, s35
; VI-NEXT: v_add_f16_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v23
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s68
+; VI-NEXT: v_mov_b32_e32 v7, s34
; VI-NEXT: v_add_f16_sdwa v7, v7, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v23
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s31
+; VI-NEXT: v_mov_b32_e32 v8, vcc_hi
; VI-NEXT: v_add_f16_sdwa v8, v8, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v23
; VI-NEXT: v_or_b32_e32 v8, v9, v8
@@ -27886,11 +28336,11 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v13, v13, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v23
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s35
+; VI-NEXT: v_mov_b32_e32 v14, s31
; VI-NEXT: v_add_f16_sdwa v14, v14, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s34, v23
+; VI-NEXT: v_add_f16_e32 v15, s30, v23
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s30
+; VI-NEXT: v_mov_b32_e32 v15, vcc_lo
; VI-NEXT: v_add_f16_sdwa v15, v15, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s91, v23
; VI-NEXT: v_or_b32_e32 v15, v16, v15
@@ -27926,11 +28376,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v24, v24, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s6, v23
; VI-NEXT: v_or_b32_e32 v23, v23, v24
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -27963,11 +28410,9 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 26
-; VI-NEXT: v_readlane_b32 s31, v32, 27
-; VI-NEXT: v_readlane_b32 s83, v32, 25
-; VI-NEXT: v_readlane_b32 s82, v32, 24
+; VI-NEXT: .LBB47_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 24
+; VI-NEXT: v_readlane_b32 s31, v32, 25
; VI-NEXT: v_readlane_b32 s81, v32, 23
; VI-NEXT: v_readlane_b32 s80, v32, 22
; VI-NEXT: v_readlane_b32 s71, v32, 21
@@ -28076,10 +28521,18 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v23, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v23 op_sel_hi:[1,0]
@@ -28105,10 +28558,8 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v21, s57, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v22, s58, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v23, s59, v23 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -28141,7 +28592,7 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -28220,11 +28671,16 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -28250,8 +28706,6 @@ define inreg <12 x i64> @bitcast_v48f16_to_v12i64_scalar(<48 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v22, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v23, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -28909,10 +29363,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v40, s34, 0
-; SI-NEXT: v_writelane_b32 v40, s35, 1
-; SI-NEXT: v_writelane_b32 v40, s30, 2
-; SI-NEXT: v_writelane_b32 v40, s31, 3
+; SI-NEXT: v_writelane_b32 v40, s30, 0
+; SI-NEXT: v_writelane_b32 v40, s31, 1
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: v_readfirstlane_b32 s13, v9
; SI-NEXT: v_readfirstlane_b32 s12, v8
@@ -28925,12 +29377,12 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s35, s13, 16
-; SI-NEXT: s_lshr_b32 s34, s11, 16
-; SI-NEXT: s_lshr_b32 s31, s9, 16
-; SI-NEXT: s_lshr_b32 s30, s7, 16
+; SI-NEXT: s_lshr_b32 s31, s13, 16
+; SI-NEXT: s_lshr_b32 s30, s11, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s9, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
; SI-NEXT: s_lshr_b32 s95, s5, 16
; SI-NEXT: s_lshr_b32 s94, s29, 16
; SI-NEXT: s_lshr_b32 s93, s27, 16
@@ -28951,8 +29403,40 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
@@ -28989,34 +29473,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -29041,10 +29499,10 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v18, s8
; SI-NEXT: v_mov_b32_e32 v20, s10
; SI-NEXT: v_mov_b32_e32 v22, s12
-; SI-NEXT: v_mov_b32_e32 v37, s35
-; SI-NEXT: v_mov_b32_e32 v38, s34
-; SI-NEXT: v_mov_b32_e32 v39, s31
-; SI-NEXT: v_mov_b32_e32 v48, s30
+; SI-NEXT: v_mov_b32_e32 v37, s31
+; SI-NEXT: v_mov_b32_e32 v38, s30
+; SI-NEXT: v_mov_b32_e32 v39, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v48, vcc_lo
; SI-NEXT: v_mov_b32_e32 v49, s95
; SI-NEXT: v_mov_b32_e32 v50, s94
; SI-NEXT: v_mov_b32_e32 v51, s93
@@ -29065,7 +29523,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s42
; SI-NEXT: v_mov_b32_e32 v25, s40
; SI-NEXT: v_mov_b32_e32 v24, s14
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -29126,7 +29584,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v37
-; SI-NEXT: v_readlane_b32 s30, v40, 2
+; SI-NEXT: v_readlane_b32 s30, v40, 0
; SI-NEXT: v_or_b32_e32 v1, v1, v35
; SI-NEXT: v_or_b32_e32 v3, v3, v34
; SI-NEXT: v_or_b32_e32 v5, v5, v33
@@ -29139,9 +29597,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v19, v19, v26
; SI-NEXT: v_or_b32_e32 v21, v21, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_readlane_b32 s31, v40, 3
-; SI-NEXT: v_readlane_b32 s35, v40, 1
-; SI-NEXT: v_readlane_b32 s34, v40, 0
+; SI-NEXT: v_readlane_b32 s31, v40, 1
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -29163,7 +29619,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s9, 16
; VI-NEXT: s_lshr_b32 s60, s8, 16
@@ -29189,8 +29645,40 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s78, s18, 16
; VI-NEXT: s_lshr_b32 s59, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s14, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s14, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[22:23], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[12:13], 1.0
@@ -29227,34 +29715,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -29303,7 +29765,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s42
; VI-NEXT: v_mov_b32_e32 v28, s41
; VI-NEXT: v_mov_b32_e32 v26, s40
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
; VI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
@@ -29369,7 +29831,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s9, 16
; GFX9-NEXT: s_lshr_b32 s60, s8, 16
@@ -29395,46 +29857,10 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s78, s18, 16
; GFX9-NEXT: s_lshr_b32 s59, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
-; GFX9-NEXT: v_add_f64 v[22:23], s[8:9], 1.0
-; GFX9-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
-; GFX9-NEXT: v_add_f64 v[18:19], s[12:13], 1.0
-; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
-; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
-; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
-; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
-; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
-; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
-; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
-; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v23
-; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v22
-; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v21
-; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v19
-; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v18
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v17
-; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v16
-; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v15
-; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v36, 16, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v11
-; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v9
-; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v52, 16, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
; GFX9-NEXT: ; implicit-def: $sgpr79
; GFX9-NEXT: ; implicit-def: $sgpr59
; GFX9-NEXT: ; implicit-def: $sgpr78
@@ -29459,8 +29885,50 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: ; implicit-def: $sgpr41
; GFX9-NEXT: ; implicit-def: $sgpr60
; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s14, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s14, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: v_add_f64 v[22:23], s[8:9], 1.0
+; GFX9-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
+; GFX9-NEXT: v_add_f64 v[18:19], s[12:13], 1.0
+; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
+; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
+; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
+; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
+; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v23
+; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v22
+; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v21
+; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v20
+; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v18
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v17
+; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v15
+; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; GFX9-NEXT: v_lshrrev_b32_e32 v36, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v11
+; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v8
+; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v52, 16, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v0
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -29509,7 +29977,7 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s42
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -29571,12 +30039,12 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s47, s6, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s56, s4, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s8, 16
@@ -29598,9 +30066,40 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
@@ -29637,34 +30136,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -29687,9 +30160,9 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s59 :: v_dual_mov_b32 v34, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s58 :: v_dual_mov_b32 v32, s14
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s57 :: v_dual_mov_b32 v30, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s56 :: v_dual_mov_b32 v28, s12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s46 :: v_dual_mov_b32 v26, s11
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s56 :: v_dual_mov_b32 v28, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s46 :: v_dual_mov_b32 v26, s10
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v53.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v55.l
@@ -29728,12 +30201,12 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s47, s6, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s56, s4, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s8, 16
@@ -29755,9 +30228,40 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[6:7], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[8:9], 1.0
@@ -29794,34 +30298,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v25, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v11, s22
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v9, s24
@@ -29845,8 +30323,8 @@ define inreg <48 x i16> @bitcast_v12f64_to_v48i16_scalar(<12 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v50, s43 :: v_dual_mov_b32 v37, s40
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s41 :: v_dual_mov_b32 v35, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s14 :: v_dual_mov_b32 v31, s13
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s12 :: v_dual_mov_b32 v29, s11
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s11 :: v_dual_mov_b32 v29, s10
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v25, 0xffff, v25
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v0
@@ -30988,10 +31466,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v24, s67, 17
; SI-NEXT: v_writelane_b32 v24, s68, 18
; SI-NEXT: v_writelane_b32 v24, s69, 19
-; SI-NEXT: v_writelane_b32 v24, s70, 20
-; SI-NEXT: v_writelane_b32 v24, s71, 21
-; SI-NEXT: v_writelane_b32 v24, s30, 22
-; SI-NEXT: v_writelane_b32 v24, s31, 23
+; SI-NEXT: v_writelane_b32 v24, s30, 20
+; SI-NEXT: v_writelane_b32 v24, s31, 21
; SI-NEXT: v_readfirstlane_b32 s7, v9
; SI-NEXT: v_readfirstlane_b32 s9, v8
; SI-NEXT: v_readfirstlane_b32 s11, v7
@@ -31008,14 +31484,14 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s89, s26, 16
; SI-NEXT: s_lshr_b32 s92, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -31028,31 +31504,31 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s93, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -31102,46 +31578,55 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
@@ -31224,8 +31709,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s59, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v24, 22
+; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v24, 20
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -31250,9 +31735,7 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v21, s57
; SI-NEXT: v_mov_b32_e32 v22, s58
; SI-NEXT: v_mov_b32_e32 v23, s59
-; SI-NEXT: v_readlane_b32 s31, v24, 23
-; SI-NEXT: v_readlane_b32 s71, v24, 21
-; SI-NEXT: v_readlane_b32 s70, v24, 20
+; SI-NEXT: v_readlane_b32 s31, v24, 21
; SI-NEXT: v_readlane_b32 s69, v24, 19
; SI-NEXT: v_readlane_b32 s68, v24, 18
; SI-NEXT: v_readlane_b32 s67, v24, 17
@@ -31278,9 +31761,6 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v48i16_to_v12f64_scalar:
; VI: ; %bb.0:
@@ -31312,10 +31792,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v24, s71, 21
; VI-NEXT: v_writelane_b32 v24, s80, 22
; VI-NEXT: v_writelane_b32 v24, s81, 23
-; VI-NEXT: v_writelane_b32 v24, s82, 24
-; VI-NEXT: v_writelane_b32 v24, s83, 25
-; VI-NEXT: v_writelane_b32 v24, s30, 26
-; VI-NEXT: v_writelane_b32 v24, s31, 27
+; VI-NEXT: v_writelane_b32 v24, s30, 24
+; VI-NEXT: v_writelane_b32 v24, s31, 25
; VI-NEXT: v_readfirstlane_b32 s7, v9
; VI-NEXT: v_readfirstlane_b32 s9, v8
; VI-NEXT: v_readfirstlane_b32 s11, v7
@@ -31325,21 +31803,21 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s77, v3
; VI-NEXT: v_readfirstlane_b32 s88, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s75, s28, 16
; VI-NEXT: s_lshr_b32 s78, s27, 16
; VI-NEXT: s_lshr_b32 s89, s26, 16
-; VI-NEXT: s_lshr_b32 s30, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s25, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -31349,40 +31827,40 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s76, s77, 16
; VI-NEXT: s_lshr_b32 s79, s88, 16
; VI-NEXT: s_lshr_b32 s90, s91, 16
-; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -31396,8 +31874,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -31426,56 +31904,65 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -31497,10 +31984,10 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
@@ -31548,8 +32035,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s59, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v24, 26
+; VI-NEXT: .LBB51_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v24, 24
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -31574,9 +32061,7 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v21, s57
; VI-NEXT: v_mov_b32_e32 v22, s58
; VI-NEXT: v_mov_b32_e32 v23, s59
-; VI-NEXT: v_readlane_b32 s31, v24, 27
-; VI-NEXT: v_readlane_b32 s83, v24, 25
-; VI-NEXT: v_readlane_b32 s82, v24, 24
+; VI-NEXT: v_readlane_b32 s31, v24, 25
; VI-NEXT: v_readlane_b32 s81, v24, 23
; VI-NEXT: v_readlane_b32 s80, v24, 22
; VI-NEXT: v_readlane_b32 s71, v24, 21
@@ -31606,9 +32091,6 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v48i16_to_v12f64_scalar:
; GFX9: ; %bb.0:
@@ -31688,10 +32170,18 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -31716,10 +32206,8 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v21, s57, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v22, s58, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -31752,7 +32240,7 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -31831,11 +32319,16 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -31861,8 +32354,6 @@ define inreg <12 x double> @bitcast_v48i16_to_v12f64_scalar(<48 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v22, s22, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v23, s23, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -32520,10 +33011,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v40, s34, 0
-; SI-NEXT: v_writelane_b32 v40, s35, 1
-; SI-NEXT: v_writelane_b32 v40, s30, 2
-; SI-NEXT: v_writelane_b32 v40, s31, 3
+; SI-NEXT: v_writelane_b32 v40, s30, 0
+; SI-NEXT: v_writelane_b32 v40, s31, 1
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: v_readfirstlane_b32 s13, v9
; SI-NEXT: v_readfirstlane_b32 s12, v8
@@ -32536,12 +33025,12 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s35, s13, 16
-; SI-NEXT: s_lshr_b32 s34, s11, 16
-; SI-NEXT: s_lshr_b32 s31, s9, 16
-; SI-NEXT: s_lshr_b32 s30, s7, 16
+; SI-NEXT: s_lshr_b32 s31, s13, 16
+; SI-NEXT: s_lshr_b32 s30, s11, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s9, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s7, 16
; SI-NEXT: s_lshr_b32 s95, s5, 16
; SI-NEXT: s_lshr_b32 s94, s29, 16
; SI-NEXT: s_lshr_b32 s93, s27, 16
@@ -32562,8 +33051,40 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[72:73], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[78:79], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[78:79], -1
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr89
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr91
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[78:79], s[78:79], exec
+; SI-NEXT: s_cselect_b32 s15, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s15, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
; SI-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
@@ -32600,34 +33121,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v5
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v3
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -32652,10 +33147,10 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v18, s8
; SI-NEXT: v_mov_b32_e32 v20, s10
; SI-NEXT: v_mov_b32_e32 v22, s12
-; SI-NEXT: v_mov_b32_e32 v37, s35
-; SI-NEXT: v_mov_b32_e32 v38, s34
-; SI-NEXT: v_mov_b32_e32 v39, s31
-; SI-NEXT: v_mov_b32_e32 v48, s30
+; SI-NEXT: v_mov_b32_e32 v37, s31
+; SI-NEXT: v_mov_b32_e32 v38, s30
+; SI-NEXT: v_mov_b32_e32 v39, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v48, vcc_lo
; SI-NEXT: v_mov_b32_e32 v49, s95
; SI-NEXT: v_mov_b32_e32 v50, s94
; SI-NEXT: v_mov_b32_e32 v51, s93
@@ -32676,7 +33171,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v26, s42
; SI-NEXT: v_mov_b32_e32 v25, s40
; SI-NEXT: v_mov_b32_e32 v24, s14
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -32737,7 +33232,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v23
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v37
-; SI-NEXT: v_readlane_b32 s30, v40, 2
+; SI-NEXT: v_readlane_b32 s30, v40, 0
; SI-NEXT: v_or_b32_e32 v1, v1, v35
; SI-NEXT: v_or_b32_e32 v3, v3, v34
; SI-NEXT: v_or_b32_e32 v5, v5, v33
@@ -32750,9 +33245,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v19, v19, v26
; SI-NEXT: v_or_b32_e32 v21, v21, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: v_readlane_b32 s31, v40, 3
-; SI-NEXT: v_readlane_b32 s35, v40, 1
-; SI-NEXT: v_readlane_b32 s34, v40, 0
+; SI-NEXT: v_readlane_b32 s31, v40, 1
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
@@ -32774,7 +33267,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s40, s9, 16
; VI-NEXT: s_lshr_b32 s60, s8, 16
@@ -32800,8 +33293,40 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; VI-NEXT: s_lshr_b32 s78, s18, 16
; VI-NEXT: s_lshr_b32 s59, s17, 16
; VI-NEXT: s_lshr_b32 s79, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[14:15], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[14:15], -1
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr41
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr40
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; VI-NEXT: s_cselect_b32 s14, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s14, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[22:23], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[18:19], s[12:13], 1.0
@@ -32838,34 +33363,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v51, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr41
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr40
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -32914,7 +33413,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v30, s42
; VI-NEXT: v_mov_b32_e32 v28, s41
; VI-NEXT: v_mov_b32_e32 v26, s40
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
; VI-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v49
@@ -32980,7 +33479,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s40, s9, 16
; GFX9-NEXT: s_lshr_b32 s60, s8, 16
@@ -33006,8 +33505,40 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s78, s18, 16
; GFX9-NEXT: s_lshr_b32 s59, s17, 16
; GFX9-NEXT: s_lshr_b32 s79, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[14:15], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[14:15], -1
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr41
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr40
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[14:15], s[14:15], exec
+; GFX9-NEXT: s_cselect_b32 s14, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s14, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[22:23], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[18:19], s[12:13], 1.0
@@ -33044,34 +33575,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr41
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr40
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -33120,7 +33625,7 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v30, s42
; GFX9-NEXT: v_mov_b32_e32 v28, s41
; GFX9-NEXT: v_mov_b32_e32 v26, s40
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -33182,12 +33687,12 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s47, s6, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s56, s4, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s8, 16
@@ -33209,9 +33714,40 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s12, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[8:9], 1.0
@@ -33248,34 +33784,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -33298,9 +33808,9 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s59 :: v_dual_mov_b32 v34, s15
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s58 :: v_dual_mov_b32 v32, s14
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s57 :: v_dual_mov_b32 v30, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s56 :: v_dual_mov_b32 v28, s12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s46 :: v_dual_mov_b32 v26, s11
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s56 :: v_dual_mov_b32 v28, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, s46 :: v_dual_mov_b32 v26, s10
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v53.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v55.l
@@ -33339,12 +33849,12 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s10, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s47, s6, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s56, s4, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s8, 16
@@ -33366,9 +33876,40 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s10
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s12, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[19:20], s[6:7], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[4:5], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[8:9], 1.0
@@ -33405,34 +33946,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v25, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v11, s22
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v9, s24
@@ -33456,8 +33971,8 @@ define inreg <48 x half> @bitcast_v12f64_to_v48f16_scalar(<12 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v50, s43 :: v_dual_mov_b32 v37, s40
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s41 :: v_dual_mov_b32 v35, s15
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, s14 :: v_dual_mov_b32 v31, s13
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s12 :: v_dual_mov_b32 v29, s11
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v30, s11 :: v_dual_mov_b32 v29, s10
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v25, 0xffff, v25
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v0
@@ -34711,10 +35226,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_writelane_b32 v32, s67, 17
; SI-NEXT: v_writelane_b32 v32, s68, 18
; SI-NEXT: v_writelane_b32 v32, s69, 19
-; SI-NEXT: v_writelane_b32 v32, s70, 20
-; SI-NEXT: v_writelane_b32 v32, s71, 21
-; SI-NEXT: v_writelane_b32 v32, s30, 22
-; SI-NEXT: v_writelane_b32 v32, s31, 23
+; SI-NEXT: v_writelane_b32 v32, s30, 20
+; SI-NEXT: v_writelane_b32 v32, s31, 21
; SI-NEXT: v_readfirstlane_b32 s6, v9
; SI-NEXT: v_readfirstlane_b32 s8, v8
; SI-NEXT: v_readfirstlane_b32 s10, v7
@@ -34731,14 +35244,14 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: s_lshr_b32 s93, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
; SI-NEXT: s_lshr_b32 s95, s24, 16
-; SI-NEXT: s_lshr_b32 s30, s23, 16
-; SI-NEXT: s_lshr_b32 s31, s22, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s35, s20, 16
-; SI-NEXT: s_lshr_b32 s68, s19, 16
-; SI-NEXT: s_lshr_b32 s69, s18, 16
-; SI-NEXT: s_lshr_b32 s70, s17, 16
-; SI-NEXT: s_lshr_b32 s71, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 s34, s19, 16
+; SI-NEXT: s_lshr_b32 s35, s18, 16
+; SI-NEXT: s_lshr_b32 s68, s17, 16
+; SI-NEXT: s_lshr_b32 s69, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -34751,31 +35264,31 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: s_lshr_b32 s91, s89, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -34825,11 +35338,20 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s59, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s71
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s69
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s68
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -34839,7 +35361,7 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s35
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -34849,20 +35371,20 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s34
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s30
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -34872,7 +35394,7 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v6, vcc_hi
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -34881,7 +35403,7 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v7, vcc_lo
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
@@ -35019,11 +35541,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v22, v23, v22
; SI-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; SI-NEXT: v_or_b32_e32 v23, v25, v23
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -35056,11 +35575,9 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 22
-; SI-NEXT: v_readlane_b32 s31, v32, 23
-; SI-NEXT: v_readlane_b32 s71, v32, 21
-; SI-NEXT: v_readlane_b32 s70, v32, 20
+; SI-NEXT: .LBB55_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 20
+; SI-NEXT: v_readlane_b32 s31, v32, 21
; SI-NEXT: v_readlane_b32 s69, v32, 19
; SI-NEXT: v_readlane_b32 s68, v32, 18
; SI-NEXT: v_readlane_b32 s67, v32, 17
@@ -35117,10 +35634,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: v_writelane_b32 v32, s71, 21
; VI-NEXT: v_writelane_b32 v32, s80, 22
; VI-NEXT: v_writelane_b32 v32, s81, 23
-; VI-NEXT: v_writelane_b32 v32, s82, 24
-; VI-NEXT: v_writelane_b32 v32, s83, 25
-; VI-NEXT: v_writelane_b32 v32, s30, 26
-; VI-NEXT: v_writelane_b32 v32, s31, 27
+; VI-NEXT: v_writelane_b32 v32, s30, 24
+; VI-NEXT: v_writelane_b32 v32, s31, 25
; VI-NEXT: v_readfirstlane_b32 s6, v9
; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s10, v7
@@ -35130,21 +35645,21 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: v_readfirstlane_b32 s76, v3
; VI-NEXT: v_readfirstlane_b32 s79, v2
; VI-NEXT: v_readfirstlane_b32 s91, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s72, s29, 16
; VI-NEXT: s_lshr_b32 s74, s28, 16
; VI-NEXT: s_lshr_b32 s77, s27, 16
; VI-NEXT: s_lshr_b32 s88, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
-; VI-NEXT: s_lshr_b32 s31, s24, 16
-; VI-NEXT: s_lshr_b32 s68, s23, 16
-; VI-NEXT: s_lshr_b32 s69, s22, 16
-; VI-NEXT: s_lshr_b32 s70, s21, 16
-; VI-NEXT: s_lshr_b32 s71, s20, 16
-; VI-NEXT: s_lshr_b32 s80, s19, 16
-; VI-NEXT: s_lshr_b32 s81, s18, 16
-; VI-NEXT: s_lshr_b32 s82, s17, 16
-; VI-NEXT: s_lshr_b32 s83, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; VI-NEXT: s_lshr_b32 s34, s23, 16
+; VI-NEXT: s_lshr_b32 s35, s22, 16
+; VI-NEXT: s_lshr_b32 s68, s21, 16
+; VI-NEXT: s_lshr_b32 s69, s20, 16
+; VI-NEXT: s_lshr_b32 s70, s19, 16
+; VI-NEXT: s_lshr_b32 s71, s18, 16
+; VI-NEXT: s_lshr_b32 s80, s17, 16
+; VI-NEXT: s_lshr_b32 s81, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -35153,38 +35668,38 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: s_lshr_b32 s75, s73, 16
; VI-NEXT: s_lshr_b32 s78, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s37, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -35201,11 +35716,11 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s50, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s51, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
@@ -35231,42 +35746,51 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v23, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s83
-; VI-NEXT: v_mov_b32_e32 v2, s82
+; VI-NEXT: v_mov_b32_e32 v0, s81
+; VI-NEXT: v_mov_b32_e32 v2, s80
; VI-NEXT: v_add_f16_sdwa v0, v0, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v23
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v23
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s81
+; VI-NEXT: v_mov_b32_e32 v2, s71
; VI-NEXT: v_add_f16_sdwa v2, v2, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v23
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s80
+; VI-NEXT: v_mov_b32_e32 v3, s70
; VI-NEXT: v_add_f16_sdwa v3, v3, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v23
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s71
+; VI-NEXT: v_mov_b32_e32 v4, s69
; VI-NEXT: v_add_f16_sdwa v4, v4, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v23
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s70
+; VI-NEXT: v_mov_b32_e32 v5, s68
; VI-NEXT: v_add_f16_sdwa v5, v5, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v23
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s69
+; VI-NEXT: v_mov_b32_e32 v6, s35
; VI-NEXT: v_add_f16_sdwa v6, v6, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v23
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s68
+; VI-NEXT: v_mov_b32_e32 v7, s34
; VI-NEXT: v_add_f16_sdwa v7, v7, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v23
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s31
+; VI-NEXT: v_mov_b32_e32 v8, vcc_hi
; VI-NEXT: v_add_f16_sdwa v8, v8, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v23
; VI-NEXT: v_or_b32_e32 v8, v9, v8
@@ -35290,11 +35814,11 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v13, v13, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v23
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s35
+; VI-NEXT: v_mov_b32_e32 v14, s31
; VI-NEXT: v_add_f16_sdwa v14, v14, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s34, v23
+; VI-NEXT: v_add_f16_e32 v15, s30, v23
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s30
+; VI-NEXT: v_mov_b32_e32 v15, vcc_lo
; VI-NEXT: v_add_f16_sdwa v15, v15, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v16, s91, v23
; VI-NEXT: v_or_b32_e32 v15, v16, v15
@@ -35330,11 +35854,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v24, v24, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s6, v23
; VI-NEXT: v_or_b32_e32 v23, v23, v24
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -35367,11 +35888,9 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 26
-; VI-NEXT: v_readlane_b32 s31, v32, 27
-; VI-NEXT: v_readlane_b32 s83, v32, 25
-; VI-NEXT: v_readlane_b32 s82, v32, 24
+; VI-NEXT: .LBB55_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 24
+; VI-NEXT: v_readlane_b32 s31, v32, 25
; VI-NEXT: v_readlane_b32 s81, v32, 23
; VI-NEXT: v_readlane_b32 s80, v32, 22
; VI-NEXT: v_readlane_b32 s71, v32, 21
@@ -35480,10 +35999,18 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s57, s57, s62
; GFX9-NEXT: s_pack_ll_b32_b16 s58, s58, s61
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s60
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v23, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v23 op_sel_hi:[1,0]
@@ -35509,10 +36036,8 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v21, s57, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v22, s58, v23 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v23, s59, v23 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -35545,7 +36070,7 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -35624,11 +36149,16 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s21, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s22, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -35654,8 +36184,6 @@ define inreg <12 x double> @bitcast_v48f16_to_v12f64_scalar(<48 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v22, 0x200, s22 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v23, 0x200, s23 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -36848,9 +37376,9 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s38, s21, 16
; SI-NEXT: s_lshr_b32 s66, s20, 16
; SI-NEXT: s_lshr_b32 s37, s19, 16
-; SI-NEXT: s_lshr_b32 s63, s18, 16
+; SI-NEXT: s_lshr_b32 s73, s18, 16
; SI-NEXT: s_lshr_b32 s36, s17, 16
-; SI-NEXT: s_lshr_b32 s61, s16, 16
+; SI-NEXT: s_lshr_b32 s63, s16, 16
; SI-NEXT: s_lshr_b32 s55, s99, 16
; SI-NEXT: s_lshr_b32 s64, s65, 16
; SI-NEXT: s_lshr_b32 s54, s96, 16
@@ -36863,26 +37391,26 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s80, s81, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s36, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s61, 16
-; SI-NEXT: s_mov_b32 s9, s61
+; SI-NEXT: s_lshl_b32 s60, s63, 16
; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
; SI-NEXT: s_lshl_b32 s7, s37, 16
; SI-NEXT: s_or_b32 s46, s4, s60
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s63, 16
-; SI-NEXT: s_mov_b32 s11, s63
+; SI-NEXT: s_lshl_b32 s62, s73, 16
+; SI-NEXT: s_mov_b32 s9, s63
; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
; SI-NEXT: s_lshl_b32 s7, s38, 16
; SI-NEXT: s_or_b32 s44, s4, s62
; SI-NEXT: s_and_b32 s4, s20, 0xffff
; SI-NEXT: s_lshl_b32 s72, s66, 16
+; SI-NEXT: s_mov_b32 s11, s73
; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s39, 16
@@ -36935,19 +37463,19 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s7, s55, 16
; SI-NEXT: s_lshr_b64 s[94:95], s[58:59], 16
; SI-NEXT: s_and_b32 s58, s98, 0xffff
-; SI-NEXT: s_or_b32 vcc_hi, s5, s7
-; SI-NEXT: s_lshl_b32 vcc_lo, s64, 16
-; SI-NEXT: s_mov_b32 s47, s61
-; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_or_b32 s31, s5, s7
+; SI-NEXT: s_lshl_b32 s30, s64, 16
; SI-NEXT: s_mov_b32 s45, s63
; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
-; SI-NEXT: s_or_b32 s58, s58, s56
-; SI-NEXT: s_lshr_b64 s[30:31], s[56:57], 16
-; SI-NEXT: s_and_b32 s56, s65, 0xffff
-; SI-NEXT: s_mov_b32 s61, s9
-; SI-NEXT: s_mov_b32 s63, s11
; SI-NEXT: s_mov_b32 s43, s73
; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
+; SI-NEXT: s_or_b32 s58, s58, s56
+; SI-NEXT: s_lshr_b64 vcc, s[56:57], 16
+; SI-NEXT: s_and_b32 s56, s65, 0xffff
+; SI-NEXT: s_mov_b32 s47, s61
+; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
+; SI-NEXT: s_mov_b32 s63, s9
+; SI-NEXT: s_mov_b32 s73, s11
; SI-NEXT: s_mov_b32 s41, s75
; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
; SI-NEXT: s_mov_b32 s15, s77
@@ -36962,11 +37490,43 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[92:93], s[92:93], 16
; SI-NEXT: s_mov_b32 s5, s59
; SI-NEXT: s_mov_b32 s59, s57
-; SI-NEXT: s_or_b32 s56, s56, vcc_lo
-; SI-NEXT: s_mov_b32 s57, vcc_hi
-; SI-NEXT: s_lshr_b64 s[34:35], vcc, 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_or_b32 s56, s56, s30
+; SI-NEXT: s_mov_b32 s57, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[30:31], 16
+; SI-NEXT: s_mov_b64 s[34:35], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[34:35], -1
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; SI-NEXT: s_cselect_b32 s61, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s61, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s65, s65, 3
; SI-NEXT: s_and_b32 s4, s65, 0xffff
; SI-NEXT: s_lshl_b32 s5, s64, 16
@@ -37057,7 +37617,7 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s43, s20, 0x30000
; SI-NEXT: s_and_b32 s18, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s63, 16
+; SI-NEXT: s_lshl_b32 s20, s73, 16
; SI-NEXT: s_or_b32 s18, s20, s18
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s44, s18, 0x30000
@@ -37067,7 +37627,7 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s45, s18, 0x30000
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s61, 16
+; SI-NEXT: s_lshl_b32 s18, s63, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s46, s16, 0x30000
@@ -37097,8 +37657,8 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[56:57], 16
+; SI-NEXT: s_lshr_b64 vcc, s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[56:57], 16
; SI-NEXT: s_lshr_b32 s36, s47, 16
; SI-NEXT: s_lshr_b32 s37, s45, 16
; SI-NEXT: s_lshr_b32 s38, s43, 16
@@ -37111,7 +37671,7 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s53, s5, 16
; SI-NEXT: s_lshr_b32 s54, s59, 16
; SI-NEXT: s_lshr_b32 s55, s57, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s16, s46, 0xffff
; SI-NEXT: s_lshl_b32 s17, s60, 16
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -37173,13 +37733,13 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s24, s53, 16
; SI-NEXT: s_or_b32 s5, s5, s24
; SI-NEXT: s_and_b32 s24, s58, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s30, 16
+; SI-NEXT: s_lshl_b32 s25, vcc_lo, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s59, 0xffff
; SI-NEXT: s_lshl_b32 s26, s54, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s34, 16
+; SI-NEXT: s_lshl_b32 s27, s30, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s57, 0xffff
; SI-NEXT: s_lshl_b32 s28, s55, 16
@@ -37249,32 +37809,6 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v48i16_to_v48f16_scalar:
; VI: ; %bb.0:
@@ -37315,10 +37849,18 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s73, s72, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s79, s79, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -37367,7 +37909,7 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s9, s9, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -37465,8 +38007,6 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v22, s8
; VI-NEXT: v_mov_b32_e32 v23, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v48i16_to_v48f16_scalar:
; GFX9: ; %bb.0:
@@ -37507,10 +38047,18 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s62, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v10
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s79, s61
; GFX9-NEXT: v_pk_add_u16 v23, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s78, s60
@@ -37583,10 +38131,8 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v21
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v22
; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v23, s79
; GFX9-NEXT: v_mov_b32_e32 v22, s78
; GFX9-NEXT: v_mov_b32_e32 v21, s77
@@ -37635,7 +38181,7 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v53, s8
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -37722,11 +38268,16 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s60, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s74, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s74, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s74, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s72, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s63, s47
; GFX11-TRUE16-NEXT: v_pk_add_u16 v23, s57, 3 op_sel_hi:[1,0]
@@ -37800,8 +38351,6 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v22
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v23
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s72 :: v_dual_mov_b32 v22, s63
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s73 :: v_dual_mov_b32 v20, s62
@@ -37892,11 +38441,16 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s60, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s74, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s74, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s74, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s72, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s63, s47
; GFX11-FAKE16-NEXT: v_pk_add_u16 v19, s57, 3 op_sel_hi:[1,0]
@@ -37970,8 +38524,6 @@ define inreg <48 x half> @bitcast_v48i16_to_v48f16_scalar(<48 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 16, v20
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v24, 16, v19
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s72 :: v_dual_mov_b32 v20, s63
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s73 :: v_dual_mov_b32 v22, s62
@@ -38896,10 +39448,18 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s74, s10, 16
; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v1, s78
; SI-NEXT: v_cvt_f32_f16_e32 v2, s77
; SI-NEXT: v_cvt_f32_f16_e32 v5, s76
@@ -39120,10 +39680,8 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_lshr_b64 v[50:51], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[48:49], v[6:7], 16
; SI-NEXT: v_lshr_b64 v[38:39], v[8:9], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v25, s72
; SI-NEXT: s_waitcnt expcnt(6)
; SI-NEXT: v_mov_b32_e32 v57, s58
@@ -39179,7 +39737,7 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v28, s61
; SI-NEXT: v_mov_b32_e32 v26, s57
; SI-NEXT: v_mov_b32_e32 v24, s47
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v54
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v47
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -39314,10 +39872,18 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: v_readfirstlane_b32 s4, v10
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v24, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v24
; VI-NEXT: v_add_f16_e32 v55, s43, v24
@@ -39367,10 +39933,8 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v25, s47, v24
; VI-NEXT: v_add_f16_e32 v23, s44, v24
; VI-NEXT: v_add_f16_e32 v24, s45, v24
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v24, s45
; VI-NEXT: v_mov_b32_e32 v23, s44
; VI-NEXT: v_mov_b32_e32 v25, s47
@@ -39419,7 +39983,7 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v55, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v55, 16, v55
; VI-NEXT: v_lshlrev_b32_e32 v54, 16, v54
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v53
@@ -39509,10 +40073,18 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s62, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v10
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s79, s61
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v23, s4, v0 op_sel_hi:[1,0]
@@ -39586,10 +40158,8 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v21
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v22
; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v23, s79
; GFX9-NEXT: v_mov_b32_e32 v22, s78
; GFX9-NEXT: v_mov_b32_e32 v21, s77
@@ -39638,7 +40208,7 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v53, s8
; GFX9-NEXT: v_mov_b32_e32 v54, s7
; GFX9-NEXT: v_mov_b32_e32 v55, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -39725,11 +40295,16 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s60, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s74, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s74, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s74, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s72, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s63, s47
; GFX11-TRUE16-NEXT: v_pk_add_f16 v23, 0x200, s57 op_sel_hi:[0,1]
@@ -39803,8 +40378,6 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v22
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v23
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s72 :: v_dual_mov_b32 v22, s63
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s73 :: v_dual_mov_b32 v20, s62
@@ -39895,11 +40468,16 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s60, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s74, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s74, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s74
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s74, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s74, s74, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s74, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s74, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s72, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s63, s47
; GFX11-FAKE16-NEXT: v_pk_add_f16 v19, 0x200, s57 op_sel_hi:[0,1]
@@ -39973,8 +40551,6 @@ define inreg <48 x i16> @bitcast_v48f16_to_v48i16_scalar(<48 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v25, 16, v20
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v24, 16, v19
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, s72 :: v_dual_mov_b32 v20, s63
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s73 :: v_dual_mov_b32 v22, s62
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
index bae36084f83f9..d6648e84e21cb 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll
@@ -198,10 +198,18 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -228,7 +236,7 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -256,8 +264,6 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v26i32_to_v26f32_scalar:
; VI: ; %bb.0:
@@ -276,10 +282,18 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -306,7 +320,7 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -334,8 +348,6 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v26i32_to_v26f32_scalar:
; GFX9: ; %bb.0:
@@ -354,10 +366,18 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -384,7 +404,7 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -412,8 +432,6 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v26i32_to_v26f32_scalar:
; GFX11: ; %bb.0:
@@ -429,11 +447,16 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -460,7 +483,7 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -476,8 +499,6 @@ define inreg <26 x float> @bitcast_v26i32_to_v26f32_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -704,10 +725,18 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
; SI-NEXT: v_add_f32_e64 v24, s60, 1.0
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -734,10 +763,8 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -770,7 +797,7 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -835,10 +862,18 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
; VI-NEXT: v_add_f32_e64 v24, s60, 1.0
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -865,10 +900,8 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -901,7 +934,7 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -966,10 +999,18 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX9-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -996,10 +1037,8 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -1032,7 +1071,7 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -1098,11 +1137,16 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX11-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -1130,8 +1174,6 @@ define inreg <26 x i32> @bitcast_v26f32_to_v26i32_scalar(<26 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1376,10 +1418,18 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1406,7 +1456,7 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1434,8 +1484,6 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v26i32_to_v13i64_scalar:
; VI: ; %bb.0:
@@ -1454,10 +1502,18 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1484,7 +1540,7 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1512,8 +1568,6 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v26i32_to_v13i64_scalar:
; GFX9: ; %bb.0:
@@ -1532,10 +1586,18 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1562,7 +1624,7 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1590,8 +1652,6 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v26i32_to_v13i64_scalar:
; GFX11: ; %bb.0:
@@ -1607,11 +1667,16 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1638,7 +1703,7 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1654,8 +1719,6 @@ define inreg <13 x i64> @bitcast_v26i32_to_v13i64_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1872,10 +1935,18 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1902,7 +1973,7 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1930,8 +2001,6 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v13i64_to_v26i32_scalar:
; VI: ; %bb.0:
@@ -1950,10 +2019,18 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -1980,7 +2057,7 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2008,8 +2085,6 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v13i64_to_v26i32_scalar:
; GFX9: ; %bb.0:
@@ -2028,10 +2103,18 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -2058,7 +2141,7 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2086,8 +2169,6 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v13i64_to_v26i32_scalar:
; GFX11: ; %bb.0:
@@ -2103,11 +2184,16 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -2134,7 +2220,7 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2150,8 +2236,6 @@ define inreg <26 x i32> @bitcast_v13i64_to_v26i32_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2361,10 +2445,18 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2391,7 +2483,7 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2419,8 +2511,6 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v26i32_to_v13f64_scalar:
; VI: ; %bb.0:
@@ -2439,10 +2529,18 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2469,7 +2567,7 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2497,8 +2595,6 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v26i32_to_v13f64_scalar:
; GFX9: ; %bb.0:
@@ -2517,10 +2613,18 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2547,7 +2651,7 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2575,8 +2679,6 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v26i32_to_v13f64_scalar:
; GFX11: ; %bb.0:
@@ -2592,11 +2694,16 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -2623,7 +2730,7 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2639,8 +2746,6 @@ define inreg <13 x double> @bitcast_v26i32_to_v13f64_scalar(<26 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2828,10 +2933,18 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -2845,10 +2958,8 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -2881,7 +2992,7 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -2946,10 +3057,18 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -2963,10 +3082,8 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -2999,7 +3116,7 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -3064,10 +3181,18 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -3081,10 +3206,8 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -3117,7 +3240,7 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -3183,11 +3306,16 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -3202,8 +3330,6 @@ define inreg <26 x i32> @bitcast_v13f64_to_v26i32_scalar(<13 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -4027,10 +4153,8 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: v_writelane_b32 v26, s36, 2
; SI-NEXT: v_writelane_b32 v26, s37, 3
; SI-NEXT: v_writelane_b32 v26, s38, 4
-; SI-NEXT: v_writelane_b32 v26, s39, 5
-; SI-NEXT: v_writelane_b32 v26, s48, 6
-; SI-NEXT: v_writelane_b32 v26, s30, 7
-; SI-NEXT: v_writelane_b32 v26, s31, 8
+; SI-NEXT: v_writelane_b32 v26, s30, 5
+; SI-NEXT: v_writelane_b32 v26, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -4045,21 +4169,21 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s92, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -4073,8 +4197,42 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -4112,75 +4270,75 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[62:63], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s41, s88, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s41
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s48, 16
+; SI-NEXT: s_lshl_b32 s41, s38, 16
; SI-NEXT: s_or_b32 s17, s17, s41
; SI-NEXT: s_lshl_b32 s41, s78, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s41
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s39, 16
+; SI-NEXT: s_lshl_b32 s41, s37, 16
; SI-NEXT: s_or_b32 s19, s19, s41
; SI-NEXT: s_lshl_b32 s41, s76, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s41
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s38, 16
+; SI-NEXT: s_lshl_b32 s41, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s41
; SI-NEXT: s_lshl_b32 s41, s74, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s41
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s37, 16
+; SI-NEXT: s_lshl_b32 s41, s35, 16
; SI-NEXT: s_or_b32 s23, s23, s41
; SI-NEXT: s_lshl_b32 s41, s72, 16
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_or_b32 s24, s24, s41
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s36, 16
+; SI-NEXT: s_lshl_b32 s41, s34, 16
; SI-NEXT: s_or_b32 s25, s25, s41
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s41, s62, 16
; SI-NEXT: s_or_b32 s26, s26, s41
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s35, 16
+; SI-NEXT: s_lshl_b32 s41, s31, 16
; SI-NEXT: s_or_b32 s27, s27, s41
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s41, s60, 16
; SI-NEXT: s_or_b32 s28, s28, s41
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s29, s29, s41
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s41, s58, 16
; SI-NEXT: s_or_b32 s14, s14, s41
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s31, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_hi, 16
; SI-NEXT: s_or_b32 s15, s15, s41
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s41, s56, 16
; SI-NEXT: s_or_b32 s12, s12, s41
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s30, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_lo, 16
; SI-NEXT: s_or_b32 s13, s13, s41
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s41, s46, 16
@@ -4206,7 +4364,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s40, s92, 16
; SI-NEXT: s_or_b32 s7, s7, s41
; SI-NEXT: s_or_b32 s5, s5, s40
-; SI-NEXT: v_readlane_b32 s30, v26, 7
+; SI-NEXT: v_readlane_b32 s30, v26, 5
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4233,9 +4391,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v23, s7
; SI-NEXT: v_mov_b32_e32 v24, s4
; SI-NEXT: v_mov_b32_e32 v25, s5
-; SI-NEXT: v_readlane_b32 s31, v26, 8
-; SI-NEXT: v_readlane_b32 s48, v26, 6
-; SI-NEXT: v_readlane_b32 s39, v26, 5
+; SI-NEXT: v_readlane_b32 s31, v26, 6
; SI-NEXT: v_readlane_b32 s38, v26, 4
; SI-NEXT: v_readlane_b32 s37, v26, 3
; SI-NEXT: v_readlane_b32 s36, v26, 2
@@ -4246,34 +4402,6 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v26i32_to_v52i16_scalar:
; VI: ; %bb.0:
@@ -4292,7 +4420,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -4320,8 +4448,42 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -4374,7 +4536,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -4480,34 +4642,6 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v26i32_to_v52i16_scalar:
; GFX9: ; %bb.0:
@@ -4526,7 +4660,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -4554,8 +4688,42 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -4608,7 +4776,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s91
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s90
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s89
@@ -4662,34 +4830,6 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v26i32_to_v52i16_scalar:
; GFX11: ; %bb.0:
@@ -4705,7 +4845,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s78, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s4, 16
; GFX11-NEXT: s_lshr_b32 s13, s5, 16
@@ -4733,9 +4873,42 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s78, -1
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -4788,7 +4961,7 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s77
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s76
@@ -4830,34 +5003,6 @@ define inreg <52 x i16> @bitcast_v26i32_to_v52i16_scalar(<26 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6056,10 +6201,8 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v26, s71, 21
; SI-NEXT: v_writelane_b32 v26, s80, 22
; SI-NEXT: v_writelane_b32 v26, s81, 23
-; SI-NEXT: v_writelane_b32 v26, s82, 24
-; SI-NEXT: v_writelane_b32 v26, s83, 25
-; SI-NEXT: v_writelane_b32 v26, s30, 26
-; SI-NEXT: v_writelane_b32 v26, s31, 27
+; SI-NEXT: v_writelane_b32 v26, s30, 24
+; SI-NEXT: v_writelane_b32 v26, s31, 25
; SI-NEXT: v_readfirstlane_b32 s7, v11
; SI-NEXT: v_readfirstlane_b32 s9, v10
; SI-NEXT: v_readfirstlane_b32 s11, v9
@@ -6070,22 +6213,22 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s78, v4
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s93, v2
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; SI-NEXT: v_readfirstlane_b32 s31, v0
; SI-NEXT: s_lshr_b32 s76, s29, 16
; SI-NEXT: s_lshr_b32 s79, s28, 16
; SI-NEXT: s_lshr_b32 s89, s27, 16
; SI-NEXT: s_lshr_b32 s91, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s31, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -6096,38 +6239,38 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s77, s78, 16
; SI-NEXT: s_lshr_b32 s88, s90, 16
; SI-NEXT: s_lshr_b32 s92, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -6144,10 +6287,10 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -6180,51 +6323,60 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
@@ -6251,14 +6403,14 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -6312,8 +6464,8 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s61, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v26, 26
+; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v26, 24
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -6340,9 +6492,7 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v23, s59
; SI-NEXT: v_mov_b32_e32 v24, s60
; SI-NEXT: v_mov_b32_e32 v25, s61
-; SI-NEXT: v_readlane_b32 s31, v26, 27
-; SI-NEXT: v_readlane_b32 s83, v26, 25
-; SI-NEXT: v_readlane_b32 s82, v26, 24
+; SI-NEXT: v_readlane_b32 s31, v26, 25
; SI-NEXT: v_readlane_b32 s81, v26, 23
; SI-NEXT: v_readlane_b32 s80, v26, 22
; SI-NEXT: v_readlane_b32 s71, v26, 21
@@ -6372,9 +6522,6 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v52i16_to_v26i32_scalar:
; VI: ; %bb.0:
@@ -6410,10 +6557,8 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v26, s83, 25
; VI-NEXT: v_writelane_b32 v26, s84, 26
; VI-NEXT: v_writelane_b32 v26, s85, 27
-; VI-NEXT: v_writelane_b32 v26, s86, 28
-; VI-NEXT: v_writelane_b32 v26, s87, 29
-; VI-NEXT: v_writelane_b32 v26, s30, 30
-; VI-NEXT: v_writelane_b32 v26, s31, 31
+; VI-NEXT: v_writelane_b32 v26, s30, 28
+; VI-NEXT: v_writelane_b32 v26, s31, 29
; VI-NEXT: v_readfirstlane_b32 s7, v11
; VI-NEXT: v_readfirstlane_b32 s9, v10
; VI-NEXT: v_readfirstlane_b32 s11, v9
@@ -6423,23 +6568,23 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s90, v3
-; VI-NEXT: v_readfirstlane_b32 s31, v2
-; VI-NEXT: v_readfirstlane_b32 s68, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; VI-NEXT: v_readfirstlane_b32 s34, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s76, s29, 16
; VI-NEXT: s_lshr_b32 s79, s28, 16
; VI-NEXT: s_lshr_b32 s89, s27, 16
; VI-NEXT: s_lshr_b32 s91, s26, 16
-; VI-NEXT: s_lshr_b32 s34, s25, 16
-; VI-NEXT: s_lshr_b32 s69, s24, 16
-; VI-NEXT: s_lshr_b32 s80, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 s30, s25, 16
+; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s70, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -6449,42 +6594,42 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s74, s75, 16
; VI-NEXT: s_lshr_b32 s77, s78, 16
; VI-NEXT: s_lshr_b32 s88, s90, 16
-; VI-NEXT: s_lshr_b32 s30, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 s68, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -6498,14 +6643,14 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s90
; VI-NEXT: s_lshl_b32 s5, s88, 16
@@ -6534,56 +6679,65 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -6605,20 +6759,20 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s90, s90, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -6666,8 +6820,8 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s61, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v26, 30
+; VI-NEXT: .LBB15_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v26, 28
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -6694,9 +6848,7 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v23, s59
; VI-NEXT: v_mov_b32_e32 v24, s60
; VI-NEXT: v_mov_b32_e32 v25, s61
-; VI-NEXT: v_readlane_b32 s31, v26, 31
-; VI-NEXT: v_readlane_b32 s87, v26, 29
-; VI-NEXT: v_readlane_b32 s86, v26, 28
+; VI-NEXT: v_readlane_b32 s31, v26, 29
; VI-NEXT: v_readlane_b32 s85, v26, 27
; VI-NEXT: v_readlane_b32 s84, v26, 26
; VI-NEXT: v_readlane_b32 s83, v26, 25
@@ -6730,9 +6882,6 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v52i16_to_v26i32_scalar:
; GFX9: ; %bb.0:
@@ -6818,10 +6967,18 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -6848,10 +7005,8 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v24, s60, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -6884,7 +7039,7 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -6969,11 +7124,16 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -7001,8 +7161,6 @@ define inreg <26 x i32> @bitcast_v52i16_to_v26i32_scalar(<52 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v24, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v25, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -7809,10 +7967,8 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: v_writelane_b32 v26, s36, 2
; SI-NEXT: v_writelane_b32 v26, s37, 3
; SI-NEXT: v_writelane_b32 v26, s38, 4
-; SI-NEXT: v_writelane_b32 v26, s39, 5
-; SI-NEXT: v_writelane_b32 v26, s48, 6
-; SI-NEXT: v_writelane_b32 v26, s30, 7
-; SI-NEXT: v_writelane_b32 v26, s31, 8
+; SI-NEXT: v_writelane_b32 v26, s30, 5
+; SI-NEXT: v_writelane_b32 v26, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -7827,21 +7983,21 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s92, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -7855,8 +8011,42 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -7894,75 +8084,75 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[62:63], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s41, s88, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s41
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s48, 16
+; SI-NEXT: s_lshl_b32 s41, s38, 16
; SI-NEXT: s_or_b32 s17, s17, s41
; SI-NEXT: s_lshl_b32 s41, s78, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s41
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s39, 16
+; SI-NEXT: s_lshl_b32 s41, s37, 16
; SI-NEXT: s_or_b32 s19, s19, s41
; SI-NEXT: s_lshl_b32 s41, s76, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s41
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s38, 16
+; SI-NEXT: s_lshl_b32 s41, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s41
; SI-NEXT: s_lshl_b32 s41, s74, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s41
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s37, 16
+; SI-NEXT: s_lshl_b32 s41, s35, 16
; SI-NEXT: s_or_b32 s23, s23, s41
; SI-NEXT: s_lshl_b32 s41, s72, 16
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_or_b32 s24, s24, s41
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s36, 16
+; SI-NEXT: s_lshl_b32 s41, s34, 16
; SI-NEXT: s_or_b32 s25, s25, s41
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s41, s62, 16
; SI-NEXT: s_or_b32 s26, s26, s41
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s35, 16
+; SI-NEXT: s_lshl_b32 s41, s31, 16
; SI-NEXT: s_or_b32 s27, s27, s41
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s41, s60, 16
; SI-NEXT: s_or_b32 s28, s28, s41
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s29, s29, s41
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s41, s58, 16
; SI-NEXT: s_or_b32 s14, s14, s41
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s31, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_hi, 16
; SI-NEXT: s_or_b32 s15, s15, s41
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s41, s56, 16
; SI-NEXT: s_or_b32 s12, s12, s41
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s30, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_lo, 16
; SI-NEXT: s_or_b32 s13, s13, s41
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s41, s46, 16
@@ -7988,7 +8178,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: s_lshl_b32 s40, s92, 16
; SI-NEXT: s_or_b32 s7, s7, s41
; SI-NEXT: s_or_b32 s5, s5, s40
-; SI-NEXT: v_readlane_b32 s30, v26, 7
+; SI-NEXT: v_readlane_b32 s30, v26, 5
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8015,9 +8205,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v23, s7
; SI-NEXT: v_mov_b32_e32 v24, s4
; SI-NEXT: v_mov_b32_e32 v25, s5
-; SI-NEXT: v_readlane_b32 s31, v26, 8
-; SI-NEXT: v_readlane_b32 s48, v26, 6
-; SI-NEXT: v_readlane_b32 s39, v26, 5
+; SI-NEXT: v_readlane_b32 s31, v26, 6
; SI-NEXT: v_readlane_b32 s38, v26, 4
; SI-NEXT: v_readlane_b32 s37, v26, 3
; SI-NEXT: v_readlane_b32 s36, v26, 2
@@ -8028,34 +8216,6 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v26i32_to_v52f16_scalar:
; VI: ; %bb.0:
@@ -8074,7 +8234,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -8102,8 +8262,42 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -8156,7 +8350,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -8262,34 +8456,6 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v26i32_to_v52f16_scalar:
; GFX9: ; %bb.0:
@@ -8308,7 +8474,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -8336,8 +8502,42 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -8390,7 +8590,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s91
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s90
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s89
@@ -8444,34 +8644,6 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v26i32_to_v52f16_scalar:
; GFX11: ; %bb.0:
@@ -8487,7 +8659,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s78, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s4, 16
; GFX11-NEXT: s_lshr_b32 s13, s5, 16
@@ -8515,9 +8687,42 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s78, -1
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -8570,7 +8775,7 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s77
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s76
@@ -8612,34 +8817,6 @@ define inreg <52 x half> @bitcast_v26i32_to_v52f16_scalar(<26 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9959,10 +10136,8 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s71, 21
; SI-NEXT: v_writelane_b32 v32, s80, 22
; SI-NEXT: v_writelane_b32 v32, s81, 23
-; SI-NEXT: v_writelane_b32 v32, s82, 24
-; SI-NEXT: v_writelane_b32 v32, s83, 25
-; SI-NEXT: v_writelane_b32 v32, s30, 26
-; SI-NEXT: v_writelane_b32 v32, s31, 27
+; SI-NEXT: v_writelane_b32 v32, s30, 24
+; SI-NEXT: v_writelane_b32 v32, s31, 25
; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: v_readfirstlane_b32 s8, v10
; SI-NEXT: v_readfirstlane_b32 s10, v9
@@ -9977,18 +10152,18 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s94, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s93, s28, 16
-; SI-NEXT: s_lshr_b32 s30, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -10003,43 +10178,43 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s95, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -10083,11 +10258,20 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s83
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s81
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s80
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -10099,7 +10283,7 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -10107,11 +10291,11 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s69
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -10122,7 +10306,7 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s68
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -10131,11 +10315,11 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s34
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -10146,7 +10330,7 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -10155,11 +10339,11 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s30
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_hi
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -10170,7 +10354,7 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -10293,11 +10477,8 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v24, v25, v24
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v26
; SI-NEXT: v_or_b32_e32 v25, v27, v25
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -10330,11 +10511,9 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 26
-; SI-NEXT: v_readlane_b32 s31, v32, 27
-; SI-NEXT: v_readlane_b32 s83, v32, 25
-; SI-NEXT: v_readlane_b32 s82, v32, 24
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 24
+; SI-NEXT: v_readlane_b32 s31, v32, 25
; SI-NEXT: v_readlane_b32 s81, v32, 23
; SI-NEXT: v_readlane_b32 s80, v32, 22
; SI-NEXT: v_readlane_b32 s71, v32, 21
@@ -10399,10 +10578,8 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s83, 25
; VI-NEXT: v_writelane_b32 v32, s84, 26
; VI-NEXT: v_writelane_b32 v32, s85, 27
-; VI-NEXT: v_writelane_b32 v32, s86, 28
-; VI-NEXT: v_writelane_b32 v32, s87, 29
-; VI-NEXT: v_writelane_b32 v32, s30, 30
-; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_writelane_b32 v32, s30, 28
+; VI-NEXT: v_writelane_b32 v32, s31, 29
; VI-NEXT: v_readfirstlane_b32 s6, v11
; VI-NEXT: v_readfirstlane_b32 s8, v10
; VI-NEXT: v_readfirstlane_b32 s10, v9
@@ -10412,23 +10589,23 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s89, v3
-; VI-NEXT: v_readfirstlane_b32 s30, v2
-; VI-NEXT: v_readfirstlane_b32 s35, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; VI-NEXT: v_readfirstlane_b32 s31, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s74, s29, 16
; VI-NEXT: s_lshr_b32 s77, s28, 16
; VI-NEXT: s_lshr_b32 s88, s27, 16
; VI-NEXT: s_lshr_b32 s90, s26, 16
-; VI-NEXT: s_lshr_b32 s31, s25, 16
-; VI-NEXT: s_lshr_b32 s68, s24, 16
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s25, 16
+; VI-NEXT: s_lshr_b32 s34, s24, 16
+; VI-NEXT: s_lshr_b32 s68, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -10438,42 +10615,42 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s76, s75, 16
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s91, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -10487,14 +10664,14 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -10523,46 +10700,55 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v25, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s87
-; VI-NEXT: v_mov_b32_e32 v2, s86
+; VI-NEXT: v_mov_b32_e32 v0, s85
+; VI-NEXT: v_mov_b32_e32 v2, s84
; VI-NEXT: v_add_f16_sdwa v0, v0, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v25
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v25
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_mov_b32_e32 v2, s83
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v25
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s84
+; VI-NEXT: v_mov_b32_e32 v3, s82
; VI-NEXT: v_add_f16_sdwa v3, v3, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v25
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s83
+; VI-NEXT: v_mov_b32_e32 v4, s81
; VI-NEXT: v_add_f16_sdwa v4, v4, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v25
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s82
+; VI-NEXT: v_mov_b32_e32 v5, s80
; VI-NEXT: v_add_f16_sdwa v5, v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v25
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s81
+; VI-NEXT: v_mov_b32_e32 v6, s71
; VI-NEXT: v_add_f16_sdwa v6, v6, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v25
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s70
+; VI-NEXT: v_mov_b32_e32 v7, s68
; VI-NEXT: v_add_f16_sdwa v7, v7, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v25
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s68
+; VI-NEXT: v_mov_b32_e32 v8, s34
; VI-NEXT: v_add_f16_sdwa v8, v8, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v25
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s31
+; VI-NEXT: v_mov_b32_e32 v9, vcc_hi
; VI-NEXT: v_add_f16_sdwa v9, v9, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v25
; VI-NEXT: v_or_b32_e32 v9, v10, v9
@@ -10582,17 +10768,17 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v13, v13, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v25
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s80
+; VI-NEXT: v_mov_b32_e32 v14, s70
; VI-NEXT: v_add_f16_sdwa v14, v14, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s71, v25
+; VI-NEXT: v_add_f16_e32 v15, s69, v25
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s69
+; VI-NEXT: v_mov_b32_e32 v15, s35
; VI-NEXT: v_add_f16_sdwa v15, v15, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s35, v25
+; VI-NEXT: v_add_f16_e32 v16, s31, v25
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_mov_b32_e32 v16, s30
; VI-NEXT: v_add_f16_sdwa v16, v16, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s30, v25
+; VI-NEXT: v_add_f16_e32 v17, vcc_lo, v25
; VI-NEXT: v_or_b32_e32 v16, v17, v16
; VI-NEXT: v_mov_b32_e32 v17, s91
; VI-NEXT: v_add_f16_sdwa v17, v17, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -10630,11 +10816,8 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v26, v26, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s6, v25
; VI-NEXT: v_or_b32_e32 v25, v25, v26
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -10667,11 +10850,9 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s31, v32, 31
-; VI-NEXT: v_readlane_b32 s87, v32, 29
-; VI-NEXT: v_readlane_b32 s86, v32, 28
+; VI-NEXT: .LBB19_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 28
+; VI-NEXT: v_readlane_b32 s31, v32, 29
; VI-NEXT: v_readlane_b32 s85, v32, 27
; VI-NEXT: v_readlane_b32 s84, v32, 26
; VI-NEXT: v_readlane_b32 s83, v32, 25
@@ -10790,10 +10971,18 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v25, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v25 op_sel_hi:[1,0]
@@ -10821,10 +11010,8 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v23, s59, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v24, s60, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v25, s61, v25 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -10857,7 +11044,7 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -10942,11 +11129,16 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -10974,8 +11166,6 @@ define inreg <26 x i32> @bitcast_v52f16_to_v26i32_scalar(<52 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v24, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v25, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11220,10 +11410,18 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
; SI-NEXT: v_add_f32_e64 v24, s60, 1.0
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -11250,10 +11448,8 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11286,7 +11482,7 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11351,10 +11547,18 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
; VI-NEXT: v_add_f32_e64 v24, s60, 1.0
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -11381,10 +11585,8 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -11417,7 +11619,7 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -11482,10 +11684,18 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX9-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -11512,10 +11722,8 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -11548,7 +11756,7 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -11614,11 +11822,16 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX11-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -11646,8 +11859,6 @@ define inreg <13 x i64> @bitcast_v26f32_to_v13i64_scalar(<26 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -11899,10 +12110,18 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -11929,7 +12148,7 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -11957,8 +12176,6 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v13i64_to_v26f32_scalar:
; VI: ; %bb.0:
@@ -11977,10 +12194,18 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -12007,7 +12232,7 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12035,8 +12260,6 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v13i64_to_v26f32_scalar:
; GFX9: ; %bb.0:
@@ -12055,10 +12278,18 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -12085,7 +12316,7 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12113,8 +12344,6 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v13i64_to_v26f32_scalar:
; GFX11: ; %bb.0:
@@ -12130,11 +12359,16 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -12161,7 +12395,7 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12177,8 +12411,6 @@ define inreg <26 x float> @bitcast_v13i64_to_v26f32_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -12405,10 +12637,18 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
; SI-NEXT: v_add_f32_e64 v24, s60, 1.0
; SI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -12435,10 +12675,8 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -12471,7 +12709,7 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12536,10 +12774,18 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
; VI-NEXT: v_add_f32_e64 v24, s60, 1.0
; VI-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -12566,10 +12812,8 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -12602,7 +12846,7 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12667,10 +12911,18 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX9-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX9-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -12697,10 +12949,8 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -12733,7 +12983,7 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -12799,11 +13049,16 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
; GFX11-NEXT: v_add_f32_e64 v24, s60, 1.0
; GFX11-NEXT: v_add_f32_e64 v23, s59, 1.0
@@ -12831,8 +13086,6 @@ define inreg <13 x double> @bitcast_v26f32_to_v13f64_scalar(<26 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -13055,10 +13308,18 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -13072,10 +13333,8 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -13108,7 +13367,7 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -13173,10 +13432,18 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -13190,10 +13457,8 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -13226,7 +13491,7 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -13291,10 +13556,18 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -13308,10 +13581,8 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -13344,7 +13615,7 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -13410,11 +13681,16 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
@@ -13429,8 +13705,6 @@ define inreg <26 x float> @bitcast_v13f64_to_v26f32_scalar(<13 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -14233,10 +14507,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v45, s36, 2
; SI-NEXT: v_writelane_b32 v45, s37, 3
; SI-NEXT: v_writelane_b32 v45, s38, 4
-; SI-NEXT: v_writelane_b32 v45, s39, 5
-; SI-NEXT: v_writelane_b32 v45, s48, 6
-; SI-NEXT: v_writelane_b32 v45, s30, 7
-; SI-NEXT: v_writelane_b32 v45, s31, 8
+; SI-NEXT: v_writelane_b32 v45, s30, 5
+; SI-NEXT: v_writelane_b32 v45, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -14251,17 +14523,17 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s48, s5, 16
-; SI-NEXT: s_lshr_b32 s39, s7, 16
-; SI-NEXT: s_lshr_b32 s38, s9, 16
-; SI-NEXT: s_lshr_b32 s37, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s35, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s31, s27, 16
-; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s38, s5, 16
+; SI-NEXT: s_lshr_b32 s37, s7, 16
+; SI-NEXT: s_lshr_b32 s36, s9, 16
+; SI-NEXT: s_lshr_b32 s35, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s31, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s25, 16
; SI-NEXT: s_lshr_b32 s95, s23, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s93, s19, 16
@@ -14279,8 +14551,42 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v25, s5, 1.0
; SI-NEXT: v_add_f32_e64 v24, s4, 1.0
; SI-NEXT: v_add_f32_e64 v23, s7, 1.0
@@ -14338,36 +14644,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -14399,15 +14677,15 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v43, s93
; SI-NEXT: v_mov_b32_e32 v42, s94
; SI-NEXT: v_mov_b32_e32 v41, s95
-; SI-NEXT: v_mov_b32_e32 v40, s30
-; SI-NEXT: v_mov_b32_e32 v55, s31
-; SI-NEXT: v_mov_b32_e32 v54, s34
-; SI-NEXT: v_mov_b32_e32 v53, s35
-; SI-NEXT: v_mov_b32_e32 v52, s36
-; SI-NEXT: v_mov_b32_e32 v51, s37
-; SI-NEXT: v_mov_b32_e32 v50, s38
-; SI-NEXT: v_mov_b32_e32 v49, s39
-; SI-NEXT: v_mov_b32_e32 v48, s48
+; SI-NEXT: v_mov_b32_e32 v40, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v55, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v53, s31
+; SI-NEXT: v_mov_b32_e32 v52, s34
+; SI-NEXT: v_mov_b32_e32 v51, s35
+; SI-NEXT: v_mov_b32_e32 v50, s36
+; SI-NEXT: v_mov_b32_e32 v49, s37
+; SI-NEXT: v_mov_b32_e32 v48, s38
; SI-NEXT: v_mov_b32_e32 v26, s40
; SI-NEXT: v_mov_b32_e32 v27, s42
; SI-NEXT: v_mov_b32_e32 v28, s44
@@ -14421,7 +14699,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v36, s76
; SI-NEXT: v_mov_b32_e32 v37, s78
; SI-NEXT: v_mov_b32_e32 v38, s88
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -14492,7 +14770,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v24, v24, v26
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v48
-; SI-NEXT: v_readlane_b32 s30, v45, 7
+; SI-NEXT: v_readlane_b32 s30, v45, 5
; SI-NEXT: v_or_b32_e32 v1, v1, v38
; SI-NEXT: v_or_b32_e32 v3, v3, v37
; SI-NEXT: v_or_b32_e32 v5, v5, v36
@@ -14506,9 +14784,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v21, v21, v28
; SI-NEXT: v_or_b32_e32 v23, v23, v27
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: v_readlane_b32 s31, v45, 8
-; SI-NEXT: v_readlane_b32 s48, v45, 6
-; SI-NEXT: v_readlane_b32 s39, v45, 5
+; SI-NEXT: v_readlane_b32 s31, v45, 6
; SI-NEXT: v_readlane_b32 s38, v45, 4
; SI-NEXT: v_readlane_b32 s37, v45, 3
; SI-NEXT: v_readlane_b32 s36, v45, 2
@@ -14541,7 +14817,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -14569,8 +14845,42 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v25, s6, 1.0
; VI-NEXT: v_add_f32_e64 v24, s7, 1.0
; VI-NEXT: v_add_f32_e64 v23, s8, 1.0
@@ -14623,36 +14933,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v41, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v42, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -14705,7 +14987,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s44
; VI-NEXT: v_mov_b32_e32 v27, s43
; VI-NEXT: v_mov_b32_e32 v26, s42
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
; VI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
; VI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
@@ -14786,7 +15068,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -14814,8 +15096,42 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v25, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v24, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v23, s8, 1.0
@@ -14868,36 +15184,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -14950,7 +15238,7 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -15023,11 +15311,11 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s8, 16
@@ -15052,9 +15340,42 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s76, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s77, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v25, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v24, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v23, s6, 1.0
@@ -15107,36 +15428,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -15162,8 +15455,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s13 :: v_dual_mov_b32 v26, s12
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v66.l
@@ -15206,11 +15499,11 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s10, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s8, 16
@@ -15235,9 +15528,42 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s76, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s77, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v21, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v22, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v23, s6, 1.0
@@ -15290,36 +15616,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v5
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_mov_b32 v4, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
@@ -15345,8 +15643,8 @@ define inreg <52 x i16> @bitcast_v26f32_to_v52i16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s13 :: v_dual_mov_b32 v26, s12
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v69, 0xffff, v0
@@ -16598,10 +16896,8 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v26, s71, 21
; SI-NEXT: v_writelane_b32 v26, s80, 22
; SI-NEXT: v_writelane_b32 v26, s81, 23
-; SI-NEXT: v_writelane_b32 v26, s82, 24
-; SI-NEXT: v_writelane_b32 v26, s83, 25
-; SI-NEXT: v_writelane_b32 v26, s30, 26
-; SI-NEXT: v_writelane_b32 v26, s31, 27
+; SI-NEXT: v_writelane_b32 v26, s30, 24
+; SI-NEXT: v_writelane_b32 v26, s31, 25
; SI-NEXT: v_readfirstlane_b32 s7, v11
; SI-NEXT: v_readfirstlane_b32 s9, v10
; SI-NEXT: v_readfirstlane_b32 s11, v9
@@ -16612,22 +16908,22 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s78, v4
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s93, v2
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; SI-NEXT: v_readfirstlane_b32 s31, v0
; SI-NEXT: s_lshr_b32 s76, s29, 16
; SI-NEXT: s_lshr_b32 s79, s28, 16
; SI-NEXT: s_lshr_b32 s89, s27, 16
; SI-NEXT: s_lshr_b32 s91, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s31, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -16638,38 +16934,38 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s77, s78, 16
; SI-NEXT: s_lshr_b32 s88, s90, 16
; SI-NEXT: s_lshr_b32 s92, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -16686,10 +16982,10 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -16722,51 +17018,60 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
@@ -16793,14 +17098,14 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -16854,8 +17159,8 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s61, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v26, 26
+; SI-NEXT: .LBB31_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v26, 24
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -16882,9 +17187,7 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v23, s59
; SI-NEXT: v_mov_b32_e32 v24, s60
; SI-NEXT: v_mov_b32_e32 v25, s61
-; SI-NEXT: v_readlane_b32 s31, v26, 27
-; SI-NEXT: v_readlane_b32 s83, v26, 25
-; SI-NEXT: v_readlane_b32 s82, v26, 24
+; SI-NEXT: v_readlane_b32 s31, v26, 25
; SI-NEXT: v_readlane_b32 s81, v26, 23
; SI-NEXT: v_readlane_b32 s80, v26, 22
; SI-NEXT: v_readlane_b32 s71, v26, 21
@@ -16914,9 +17217,6 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v52i16_to_v26f32_scalar:
; VI: ; %bb.0:
@@ -16952,10 +17252,8 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v26, s83, 25
; VI-NEXT: v_writelane_b32 v26, s84, 26
; VI-NEXT: v_writelane_b32 v26, s85, 27
-; VI-NEXT: v_writelane_b32 v26, s86, 28
-; VI-NEXT: v_writelane_b32 v26, s87, 29
-; VI-NEXT: v_writelane_b32 v26, s30, 30
-; VI-NEXT: v_writelane_b32 v26, s31, 31
+; VI-NEXT: v_writelane_b32 v26, s30, 28
+; VI-NEXT: v_writelane_b32 v26, s31, 29
; VI-NEXT: v_readfirstlane_b32 s7, v11
; VI-NEXT: v_readfirstlane_b32 s9, v10
; VI-NEXT: v_readfirstlane_b32 s11, v9
@@ -16965,23 +17263,23 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s90, v3
-; VI-NEXT: v_readfirstlane_b32 s31, v2
-; VI-NEXT: v_readfirstlane_b32 s68, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; VI-NEXT: v_readfirstlane_b32 s34, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s76, s29, 16
; VI-NEXT: s_lshr_b32 s79, s28, 16
; VI-NEXT: s_lshr_b32 s89, s27, 16
; VI-NEXT: s_lshr_b32 s91, s26, 16
-; VI-NEXT: s_lshr_b32 s34, s25, 16
-; VI-NEXT: s_lshr_b32 s69, s24, 16
-; VI-NEXT: s_lshr_b32 s80, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 s30, s25, 16
+; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s70, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -16991,42 +17289,42 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s74, s75, 16
; VI-NEXT: s_lshr_b32 s77, s78, 16
; VI-NEXT: s_lshr_b32 s88, s90, 16
-; VI-NEXT: s_lshr_b32 s30, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 s68, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -17040,14 +17338,14 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s90
; VI-NEXT: s_lshl_b32 s5, s88, 16
@@ -17076,56 +17374,65 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -17147,20 +17454,20 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s90, s90, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -17208,8 +17515,8 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s61, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v26, 30
+; VI-NEXT: .LBB31_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v26, 28
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -17236,9 +17543,7 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v23, s59
; VI-NEXT: v_mov_b32_e32 v24, s60
; VI-NEXT: v_mov_b32_e32 v25, s61
-; VI-NEXT: v_readlane_b32 s31, v26, 31
-; VI-NEXT: v_readlane_b32 s87, v26, 29
-; VI-NEXT: v_readlane_b32 s86, v26, 28
+; VI-NEXT: v_readlane_b32 s31, v26, 29
; VI-NEXT: v_readlane_b32 s85, v26, 27
; VI-NEXT: v_readlane_b32 s84, v26, 26
; VI-NEXT: v_readlane_b32 s83, v26, 25
@@ -17272,9 +17577,6 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v52i16_to_v26f32_scalar:
; GFX9: ; %bb.0:
@@ -17360,10 +17662,18 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -17390,10 +17700,8 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v24, s60, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -17426,7 +17734,7 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -17511,11 +17819,16 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -17543,8 +17856,6 @@ define inreg <26 x float> @bitcast_v52i16_to_v26f32_scalar(<52 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v24, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v25, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -18330,10 +18641,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v45, s36, 2
; SI-NEXT: v_writelane_b32 v45, s37, 3
; SI-NEXT: v_writelane_b32 v45, s38, 4
-; SI-NEXT: v_writelane_b32 v45, s39, 5
-; SI-NEXT: v_writelane_b32 v45, s48, 6
-; SI-NEXT: v_writelane_b32 v45, s30, 7
-; SI-NEXT: v_writelane_b32 v45, s31, 8
+; SI-NEXT: v_writelane_b32 v45, s30, 5
+; SI-NEXT: v_writelane_b32 v45, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -18348,17 +18657,17 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s48, s5, 16
-; SI-NEXT: s_lshr_b32 s39, s7, 16
-; SI-NEXT: s_lshr_b32 s38, s9, 16
-; SI-NEXT: s_lshr_b32 s37, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s35, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s31, s27, 16
-; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s38, s5, 16
+; SI-NEXT: s_lshr_b32 s37, s7, 16
+; SI-NEXT: s_lshr_b32 s36, s9, 16
+; SI-NEXT: s_lshr_b32 s35, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s31, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s25, 16
; SI-NEXT: s_lshr_b32 s95, s23, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s93, s19, 16
@@ -18376,8 +18685,42 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v25, s5, 1.0
; SI-NEXT: v_add_f32_e64 v24, s4, 1.0
; SI-NEXT: v_add_f32_e64 v23, s7, 1.0
@@ -18435,36 +18778,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -18496,15 +18811,15 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v43, s93
; SI-NEXT: v_mov_b32_e32 v42, s94
; SI-NEXT: v_mov_b32_e32 v41, s95
-; SI-NEXT: v_mov_b32_e32 v40, s30
-; SI-NEXT: v_mov_b32_e32 v55, s31
-; SI-NEXT: v_mov_b32_e32 v54, s34
-; SI-NEXT: v_mov_b32_e32 v53, s35
-; SI-NEXT: v_mov_b32_e32 v52, s36
-; SI-NEXT: v_mov_b32_e32 v51, s37
-; SI-NEXT: v_mov_b32_e32 v50, s38
-; SI-NEXT: v_mov_b32_e32 v49, s39
-; SI-NEXT: v_mov_b32_e32 v48, s48
+; SI-NEXT: v_mov_b32_e32 v40, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v55, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v53, s31
+; SI-NEXT: v_mov_b32_e32 v52, s34
+; SI-NEXT: v_mov_b32_e32 v51, s35
+; SI-NEXT: v_mov_b32_e32 v50, s36
+; SI-NEXT: v_mov_b32_e32 v49, s37
+; SI-NEXT: v_mov_b32_e32 v48, s38
; SI-NEXT: v_mov_b32_e32 v26, s40
; SI-NEXT: v_mov_b32_e32 v27, s42
; SI-NEXT: v_mov_b32_e32 v28, s44
@@ -18518,7 +18833,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v36, s76
; SI-NEXT: v_mov_b32_e32 v37, s78
; SI-NEXT: v_mov_b32_e32 v38, s88
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -18589,7 +18904,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v24, v24, v26
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v48
-; SI-NEXT: v_readlane_b32 s30, v45, 7
+; SI-NEXT: v_readlane_b32 s30, v45, 5
; SI-NEXT: v_or_b32_e32 v1, v1, v38
; SI-NEXT: v_or_b32_e32 v3, v3, v37
; SI-NEXT: v_or_b32_e32 v5, v5, v36
@@ -18603,9 +18918,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v21, v21, v28
; SI-NEXT: v_or_b32_e32 v23, v23, v27
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: v_readlane_b32 s31, v45, 8
-; SI-NEXT: v_readlane_b32 s48, v45, 6
-; SI-NEXT: v_readlane_b32 s39, v45, 5
+; SI-NEXT: v_readlane_b32 s31, v45, 6
; SI-NEXT: v_readlane_b32 s38, v45, 4
; SI-NEXT: v_readlane_b32 s37, v45, 3
; SI-NEXT: v_readlane_b32 s36, v45, 2
@@ -18638,7 +18951,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -18666,8 +18979,42 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v25, s6, 1.0
; VI-NEXT: v_add_f32_e64 v24, s7, 1.0
; VI-NEXT: v_add_f32_e64 v23, s8, 1.0
@@ -18720,36 +19067,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v41, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v42, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -18802,7 +19121,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s44
; VI-NEXT: v_mov_b32_e32 v27, s43
; VI-NEXT: v_mov_b32_e32 v26, s42
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
; VI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
; VI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
@@ -18883,7 +19202,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -18911,8 +19230,42 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v25, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v24, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v23, s8, 1.0
@@ -18965,36 +19318,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v43, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -19047,7 +19372,7 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s44
; GFX9-NEXT: v_mov_b32_e32 v27, s43
; GFX9-NEXT: v_mov_b32_e32 v26, s42
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -19120,11 +19445,11 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s8, 16
@@ -19149,9 +19474,42 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s76, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s77, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v25, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v24, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v23, s6, 1.0
@@ -19204,36 +19562,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -19259,8 +19589,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s13 :: v_dual_mov_b32 v26, s12
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v66.l
@@ -19303,11 +19633,11 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s10, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s8, 16
@@ -19332,9 +19662,42 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s76, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s77, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v21, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v22, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v23, s6, 1.0
@@ -19387,36 +19750,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v5
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_mov_b32 v4, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
@@ -19442,8 +19777,8 @@ define inreg <52 x half> @bitcast_v26f32_to_v52f16_scalar(<26 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s14 :: v_dual_mov_b32 v26, s13
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s13 :: v_dual_mov_b32 v26, s12
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v69, 0xffff, v0
@@ -20816,10 +21151,8 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s71, 21
; SI-NEXT: v_writelane_b32 v32, s80, 22
; SI-NEXT: v_writelane_b32 v32, s81, 23
-; SI-NEXT: v_writelane_b32 v32, s82, 24
-; SI-NEXT: v_writelane_b32 v32, s83, 25
-; SI-NEXT: v_writelane_b32 v32, s30, 26
-; SI-NEXT: v_writelane_b32 v32, s31, 27
+; SI-NEXT: v_writelane_b32 v32, s30, 24
+; SI-NEXT: v_writelane_b32 v32, s31, 25
; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: v_readfirstlane_b32 s8, v10
; SI-NEXT: v_readfirstlane_b32 s10, v9
@@ -20834,18 +21167,18 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s94, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s93, s28, 16
-; SI-NEXT: s_lshr_b32 s30, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -20860,43 +21193,43 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s95, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -20940,11 +21273,20 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s83
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s81
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s80
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -20956,7 +21298,7 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -20964,11 +21306,11 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s69
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -20979,7 +21321,7 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s68
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -20988,11 +21330,11 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s34
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -21003,7 +21345,7 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -21012,11 +21354,11 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s30
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_hi
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -21027,7 +21369,7 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -21150,11 +21492,8 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v24, v25, v24
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v26
; SI-NEXT: v_or_b32_e32 v25, v27, v25
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -21187,11 +21526,9 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 26
-; SI-NEXT: v_readlane_b32 s31, v32, 27
-; SI-NEXT: v_readlane_b32 s83, v32, 25
-; SI-NEXT: v_readlane_b32 s82, v32, 24
+; SI-NEXT: .LBB35_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 24
+; SI-NEXT: v_readlane_b32 s31, v32, 25
; SI-NEXT: v_readlane_b32 s81, v32, 23
; SI-NEXT: v_readlane_b32 s80, v32, 22
; SI-NEXT: v_readlane_b32 s71, v32, 21
@@ -21256,10 +21593,8 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: v_writelane_b32 v32, s83, 25
; VI-NEXT: v_writelane_b32 v32, s84, 26
; VI-NEXT: v_writelane_b32 v32, s85, 27
-; VI-NEXT: v_writelane_b32 v32, s86, 28
-; VI-NEXT: v_writelane_b32 v32, s87, 29
-; VI-NEXT: v_writelane_b32 v32, s30, 30
-; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_writelane_b32 v32, s30, 28
+; VI-NEXT: v_writelane_b32 v32, s31, 29
; VI-NEXT: v_readfirstlane_b32 s6, v11
; VI-NEXT: v_readfirstlane_b32 s8, v10
; VI-NEXT: v_readfirstlane_b32 s10, v9
@@ -21269,23 +21604,23 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s89, v3
-; VI-NEXT: v_readfirstlane_b32 s30, v2
-; VI-NEXT: v_readfirstlane_b32 s35, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; VI-NEXT: v_readfirstlane_b32 s31, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s74, s29, 16
; VI-NEXT: s_lshr_b32 s77, s28, 16
; VI-NEXT: s_lshr_b32 s88, s27, 16
; VI-NEXT: s_lshr_b32 s90, s26, 16
-; VI-NEXT: s_lshr_b32 s31, s25, 16
-; VI-NEXT: s_lshr_b32 s68, s24, 16
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s25, 16
+; VI-NEXT: s_lshr_b32 s34, s24, 16
+; VI-NEXT: s_lshr_b32 s68, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -21295,42 +21630,42 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: s_lshr_b32 s76, s75, 16
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s91, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -21344,14 +21679,14 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -21380,46 +21715,55 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v25, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s87
-; VI-NEXT: v_mov_b32_e32 v2, s86
+; VI-NEXT: v_mov_b32_e32 v0, s85
+; VI-NEXT: v_mov_b32_e32 v2, s84
; VI-NEXT: v_add_f16_sdwa v0, v0, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v25
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v25
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_mov_b32_e32 v2, s83
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v25
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s84
+; VI-NEXT: v_mov_b32_e32 v3, s82
; VI-NEXT: v_add_f16_sdwa v3, v3, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v25
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s83
+; VI-NEXT: v_mov_b32_e32 v4, s81
; VI-NEXT: v_add_f16_sdwa v4, v4, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v25
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s82
+; VI-NEXT: v_mov_b32_e32 v5, s80
; VI-NEXT: v_add_f16_sdwa v5, v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v25
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s81
+; VI-NEXT: v_mov_b32_e32 v6, s71
; VI-NEXT: v_add_f16_sdwa v6, v6, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v25
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s70
+; VI-NEXT: v_mov_b32_e32 v7, s68
; VI-NEXT: v_add_f16_sdwa v7, v7, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v25
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s68
+; VI-NEXT: v_mov_b32_e32 v8, s34
; VI-NEXT: v_add_f16_sdwa v8, v8, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v25
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s31
+; VI-NEXT: v_mov_b32_e32 v9, vcc_hi
; VI-NEXT: v_add_f16_sdwa v9, v9, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v25
; VI-NEXT: v_or_b32_e32 v9, v10, v9
@@ -21439,17 +21783,17 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v13, v13, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v25
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s80
+; VI-NEXT: v_mov_b32_e32 v14, s70
; VI-NEXT: v_add_f16_sdwa v14, v14, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s71, v25
+; VI-NEXT: v_add_f16_e32 v15, s69, v25
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s69
+; VI-NEXT: v_mov_b32_e32 v15, s35
; VI-NEXT: v_add_f16_sdwa v15, v15, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s35, v25
+; VI-NEXT: v_add_f16_e32 v16, s31, v25
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_mov_b32_e32 v16, s30
; VI-NEXT: v_add_f16_sdwa v16, v16, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s30, v25
+; VI-NEXT: v_add_f16_e32 v17, vcc_lo, v25
; VI-NEXT: v_or_b32_e32 v16, v17, v16
; VI-NEXT: v_mov_b32_e32 v17, s91
; VI-NEXT: v_add_f16_sdwa v17, v17, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -21487,11 +21831,8 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v26, v26, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s6, v25
; VI-NEXT: v_or_b32_e32 v25, v25, v26
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -21524,11 +21865,9 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s31, v32, 31
-; VI-NEXT: v_readlane_b32 s87, v32, 29
-; VI-NEXT: v_readlane_b32 s86, v32, 28
+; VI-NEXT: .LBB35_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 28
+; VI-NEXT: v_readlane_b32 s31, v32, 29
; VI-NEXT: v_readlane_b32 s85, v32, 27
; VI-NEXT: v_readlane_b32 s84, v32, 26
; VI-NEXT: v_readlane_b32 s83, v32, 25
@@ -21647,10 +21986,18 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v25, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v25 op_sel_hi:[1,0]
@@ -21678,10 +22025,8 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v23, s59, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v24, s60, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v25, s61, v25 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -21714,7 +22059,7 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -21799,11 +22144,16 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -21831,8 +22181,6 @@ define inreg <26 x float> @bitcast_v52f16_to_v26f32_scalar(<52 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v24, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v25, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -22067,10 +22415,18 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s40, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s41, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -22097,7 +22453,7 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -22125,8 +22481,6 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v24, s7
; SI-NEXT: v_mov_b32_e32 v25, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v13i64_to_v13f64_scalar:
; VI: ; %bb.0:
@@ -22145,10 +22499,18 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -22175,7 +22537,7 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -22203,8 +22565,6 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v13i64_to_v13f64_scalar:
; GFX9: ; %bb.0:
@@ -22223,10 +22583,18 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -22253,7 +22621,7 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -22281,8 +22649,6 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v13i64_to_v13f64_scalar:
; GFX11: ; %bb.0:
@@ -22298,11 +22664,16 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s12, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s12, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s12, s12, exec_lo
+; GFX11-NEXT: s_cselect_b32 s12, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s12, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -22329,7 +22700,7 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -22344,8 +22715,6 @@ define inreg <13 x double> @bitcast_v13i64_to_v13f64_scalar(<13 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -22533,10 +22902,18 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -22550,10 +22927,8 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; SI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -22586,7 +22961,7 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -22651,10 +23026,18 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -22668,10 +23051,8 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; VI-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -22704,7 +23085,7 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -22769,10 +23150,18 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -22786,10 +23175,8 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[20:21], s[56:57], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -22822,7 +23209,7 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -22888,11 +23275,16 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -22907,8 +23299,6 @@ define inreg <13 x i64> @bitcast_v13f64_to_v13i64_scalar(<13 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -23746,10 +24136,8 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: v_writelane_b32 v26, s36, 2
; SI-NEXT: v_writelane_b32 v26, s37, 3
; SI-NEXT: v_writelane_b32 v26, s38, 4
-; SI-NEXT: v_writelane_b32 v26, s39, 5
-; SI-NEXT: v_writelane_b32 v26, s48, 6
-; SI-NEXT: v_writelane_b32 v26, s30, 7
-; SI-NEXT: v_writelane_b32 v26, s31, 8
+; SI-NEXT: v_writelane_b32 v26, s30, 5
+; SI-NEXT: v_writelane_b32 v26, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -23764,21 +24152,21 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s92, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -23792,8 +24180,42 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -23824,15 +24246,15 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -23846,60 +24268,60 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s41, s88, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s41
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s48, 16
+; SI-NEXT: s_lshl_b32 s41, s38, 16
; SI-NEXT: s_or_b32 s17, s17, s41
; SI-NEXT: s_lshl_b32 s41, s78, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s41
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s39, 16
+; SI-NEXT: s_lshl_b32 s41, s37, 16
; SI-NEXT: s_or_b32 s19, s19, s41
; SI-NEXT: s_lshl_b32 s41, s76, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s41
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s38, 16
+; SI-NEXT: s_lshl_b32 s41, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s41
; SI-NEXT: s_lshl_b32 s41, s74, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s41
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s37, 16
+; SI-NEXT: s_lshl_b32 s41, s35, 16
; SI-NEXT: s_or_b32 s23, s23, s41
; SI-NEXT: s_lshl_b32 s41, s72, 16
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_or_b32 s24, s24, s41
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s36, 16
+; SI-NEXT: s_lshl_b32 s41, s34, 16
; SI-NEXT: s_or_b32 s25, s25, s41
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s41, s62, 16
; SI-NEXT: s_or_b32 s26, s26, s41
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s35, 16
+; SI-NEXT: s_lshl_b32 s41, s31, 16
; SI-NEXT: s_or_b32 s27, s27, s41
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s41, s60, 16
; SI-NEXT: s_or_b32 s28, s28, s41
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s29, s29, s41
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s41, s58, 16
; SI-NEXT: s_or_b32 s14, s14, s41
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s31, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_hi, 16
; SI-NEXT: s_or_b32 s15, s15, s41
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s41, s56, 16
; SI-NEXT: s_or_b32 s12, s12, s41
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s30, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_lo, 16
; SI-NEXT: s_or_b32 s13, s13, s41
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s41, s46, 16
@@ -23925,7 +24347,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s40, s92, 16
; SI-NEXT: s_or_b32 s7, s7, s41
; SI-NEXT: s_or_b32 s5, s5, s40
-; SI-NEXT: v_readlane_b32 s30, v26, 7
+; SI-NEXT: v_readlane_b32 s30, v26, 5
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -23952,9 +24374,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v23, s7
; SI-NEXT: v_mov_b32_e32 v24, s4
; SI-NEXT: v_mov_b32_e32 v25, s5
-; SI-NEXT: v_readlane_b32 s31, v26, 8
-; SI-NEXT: v_readlane_b32 s48, v26, 6
-; SI-NEXT: v_readlane_b32 s39, v26, 5
+; SI-NEXT: v_readlane_b32 s31, v26, 6
; SI-NEXT: v_readlane_b32 s38, v26, 4
; SI-NEXT: v_readlane_b32 s37, v26, 3
; SI-NEXT: v_readlane_b32 s36, v26, 2
@@ -23965,34 +24385,6 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v13i64_to_v52i16_scalar:
; VI: ; %bb.0:
@@ -24011,7 +24403,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -24039,8 +24431,42 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -24093,7 +24519,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -24199,34 +24625,6 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v13i64_to_v52i16_scalar:
; GFX9: ; %bb.0:
@@ -24245,7 +24643,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -24273,8 +24671,42 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -24327,7 +24759,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s91
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s90
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s89
@@ -24381,34 +24813,6 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v13i64_to_v52i16_scalar:
; GFX11: ; %bb.0:
@@ -24424,7 +24828,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s78, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s4, 16
; GFX11-NEXT: s_lshr_b32 s13, s5, 16
@@ -24452,9 +24856,42 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s78, -1
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -24507,7 +24944,7 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s77
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s76
@@ -24549,34 +24986,6 @@ define inreg <52 x i16> @bitcast_v13i64_to_v52i16_scalar(<13 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -25775,10 +26184,8 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v26, s71, 21
; SI-NEXT: v_writelane_b32 v26, s80, 22
; SI-NEXT: v_writelane_b32 v26, s81, 23
-; SI-NEXT: v_writelane_b32 v26, s82, 24
-; SI-NEXT: v_writelane_b32 v26, s83, 25
-; SI-NEXT: v_writelane_b32 v26, s30, 26
-; SI-NEXT: v_writelane_b32 v26, s31, 27
+; SI-NEXT: v_writelane_b32 v26, s30, 24
+; SI-NEXT: v_writelane_b32 v26, s31, 25
; SI-NEXT: v_readfirstlane_b32 s7, v11
; SI-NEXT: v_readfirstlane_b32 s9, v10
; SI-NEXT: v_readfirstlane_b32 s11, v9
@@ -25789,22 +26196,22 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s78, v4
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s93, v2
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; SI-NEXT: v_readfirstlane_b32 s31, v0
; SI-NEXT: s_lshr_b32 s76, s29, 16
; SI-NEXT: s_lshr_b32 s79, s28, 16
; SI-NEXT: s_lshr_b32 s89, s27, 16
; SI-NEXT: s_lshr_b32 s91, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s31, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -25815,38 +26222,38 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s77, s78, 16
; SI-NEXT: s_lshr_b32 s88, s90, 16
; SI-NEXT: s_lshr_b32 s92, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -25863,10 +26270,10 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -25899,51 +26306,60 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
@@ -25970,14 +26386,14 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -26031,8 +26447,8 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s61, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v26, 26
+; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v26, 24
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -26059,9 +26475,7 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v23, s59
; SI-NEXT: v_mov_b32_e32 v24, s60
; SI-NEXT: v_mov_b32_e32 v25, s61
-; SI-NEXT: v_readlane_b32 s31, v26, 27
-; SI-NEXT: v_readlane_b32 s83, v26, 25
-; SI-NEXT: v_readlane_b32 s82, v26, 24
+; SI-NEXT: v_readlane_b32 s31, v26, 25
; SI-NEXT: v_readlane_b32 s81, v26, 23
; SI-NEXT: v_readlane_b32 s80, v26, 22
; SI-NEXT: v_readlane_b32 s71, v26, 21
@@ -26091,9 +26505,6 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v52i16_to_v13i64_scalar:
; VI: ; %bb.0:
@@ -26129,10 +26540,8 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v26, s83, 25
; VI-NEXT: v_writelane_b32 v26, s84, 26
; VI-NEXT: v_writelane_b32 v26, s85, 27
-; VI-NEXT: v_writelane_b32 v26, s86, 28
-; VI-NEXT: v_writelane_b32 v26, s87, 29
-; VI-NEXT: v_writelane_b32 v26, s30, 30
-; VI-NEXT: v_writelane_b32 v26, s31, 31
+; VI-NEXT: v_writelane_b32 v26, s30, 28
+; VI-NEXT: v_writelane_b32 v26, s31, 29
; VI-NEXT: v_readfirstlane_b32 s7, v11
; VI-NEXT: v_readfirstlane_b32 s9, v10
; VI-NEXT: v_readfirstlane_b32 s11, v9
@@ -26142,23 +26551,23 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s90, v3
-; VI-NEXT: v_readfirstlane_b32 s31, v2
-; VI-NEXT: v_readfirstlane_b32 s68, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; VI-NEXT: v_readfirstlane_b32 s34, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s76, s29, 16
; VI-NEXT: s_lshr_b32 s79, s28, 16
; VI-NEXT: s_lshr_b32 s89, s27, 16
; VI-NEXT: s_lshr_b32 s91, s26, 16
-; VI-NEXT: s_lshr_b32 s34, s25, 16
-; VI-NEXT: s_lshr_b32 s69, s24, 16
-; VI-NEXT: s_lshr_b32 s80, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 s30, s25, 16
+; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s70, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -26168,42 +26577,42 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_lshr_b32 s74, s75, 16
; VI-NEXT: s_lshr_b32 s77, s78, 16
; VI-NEXT: s_lshr_b32 s88, s90, 16
-; VI-NEXT: s_lshr_b32 s30, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 s68, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -26217,14 +26626,14 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s90
; VI-NEXT: s_lshl_b32 s5, s88, 16
@@ -26253,56 +26662,65 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -26324,20 +26742,20 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s90, s90, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -26385,8 +26803,8 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s61, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v26, 30
+; VI-NEXT: .LBB43_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v26, 28
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -26413,9 +26831,7 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v23, s59
; VI-NEXT: v_mov_b32_e32 v24, s60
; VI-NEXT: v_mov_b32_e32 v25, s61
-; VI-NEXT: v_readlane_b32 s31, v26, 31
-; VI-NEXT: v_readlane_b32 s87, v26, 29
-; VI-NEXT: v_readlane_b32 s86, v26, 28
+; VI-NEXT: v_readlane_b32 s31, v26, 29
; VI-NEXT: v_readlane_b32 s85, v26, 27
; VI-NEXT: v_readlane_b32 s84, v26, 26
; VI-NEXT: v_readlane_b32 s83, v26, 25
@@ -26449,9 +26865,6 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v52i16_to_v13i64_scalar:
; GFX9: ; %bb.0:
@@ -26537,10 +26950,18 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -26567,10 +26988,8 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v24, s60, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -26603,7 +27022,7 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -26688,11 +27107,16 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -26720,8 +27144,6 @@ define inreg <13 x i64> @bitcast_v52i16_to_v13i64_scalar(<52 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v24, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v25, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -27542,10 +27964,8 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: v_writelane_b32 v26, s36, 2
; SI-NEXT: v_writelane_b32 v26, s37, 3
; SI-NEXT: v_writelane_b32 v26, s38, 4
-; SI-NEXT: v_writelane_b32 v26, s39, 5
-; SI-NEXT: v_writelane_b32 v26, s48, 6
-; SI-NEXT: v_writelane_b32 v26, s30, 7
-; SI-NEXT: v_writelane_b32 v26, s31, 8
+; SI-NEXT: v_writelane_b32 v26, s30, 5
+; SI-NEXT: v_writelane_b32 v26, s31, 6
; SI-NEXT: v_readfirstlane_b32 s14, v12
; SI-NEXT: v_readfirstlane_b32 s5, v11
; SI-NEXT: v_readfirstlane_b32 s4, v10
@@ -27560,21 +27980,21 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s15, v1
; SI-NEXT: s_cmp_lg_u32 s14, 0
; SI-NEXT: v_readfirstlane_b32 s14, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s92, s5, 16
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -27588,8 +28008,42 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -27620,15 +28074,15 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: s_lshr_b32 s93, s7, 16
; SI-NEXT: s_lshr_b32 s94, s9, 16
; SI-NEXT: s_lshr_b32 s95, s11, 16
-; SI-NEXT: s_lshr_b32 s30, s13, 16
-; SI-NEXT: s_lshr_b32 s31, s15, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s37, s23, 16
-; SI-NEXT: s_lshr_b32 s38, s21, 16
-; SI-NEXT: s_lshr_b32 s39, s19, 16
-; SI-NEXT: s_lshr_b32 s48, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s13, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s15, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s35, s23, 16
+; SI-NEXT: s_lshr_b32 s36, s21, 16
+; SI-NEXT: s_lshr_b32 s37, s19, 16
+; SI-NEXT: s_lshr_b32 s38, s17, 16
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[42:43], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[8:9], 16
@@ -27642,60 +28096,60 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s41, s88, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s41
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s48, 16
+; SI-NEXT: s_lshl_b32 s41, s38, 16
; SI-NEXT: s_or_b32 s17, s17, s41
; SI-NEXT: s_lshl_b32 s41, s78, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s41
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s39, 16
+; SI-NEXT: s_lshl_b32 s41, s37, 16
; SI-NEXT: s_or_b32 s19, s19, s41
; SI-NEXT: s_lshl_b32 s41, s76, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s41
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s38, 16
+; SI-NEXT: s_lshl_b32 s41, s36, 16
; SI-NEXT: s_or_b32 s21, s21, s41
; SI-NEXT: s_lshl_b32 s41, s74, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s41
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s37, 16
+; SI-NEXT: s_lshl_b32 s41, s35, 16
; SI-NEXT: s_or_b32 s23, s23, s41
; SI-NEXT: s_lshl_b32 s41, s72, 16
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_or_b32 s24, s24, s41
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s36, 16
+; SI-NEXT: s_lshl_b32 s41, s34, 16
; SI-NEXT: s_or_b32 s25, s25, s41
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s41, s62, 16
; SI-NEXT: s_or_b32 s26, s26, s41
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s35, 16
+; SI-NEXT: s_lshl_b32 s41, s31, 16
; SI-NEXT: s_or_b32 s27, s27, s41
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s41, s60, 16
; SI-NEXT: s_or_b32 s28, s28, s41
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s34, 16
+; SI-NEXT: s_lshl_b32 s41, s30, 16
; SI-NEXT: s_or_b32 s29, s29, s41
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s41, s58, 16
; SI-NEXT: s_or_b32 s14, s14, s41
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s31, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_hi, 16
; SI-NEXT: s_or_b32 s15, s15, s41
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s41, s56, 16
; SI-NEXT: s_or_b32 s12, s12, s41
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s30, 16
+; SI-NEXT: s_lshl_b32 s41, vcc_lo, 16
; SI-NEXT: s_or_b32 s13, s13, s41
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s41, s46, 16
@@ -27721,7 +28175,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: s_lshl_b32 s40, s92, 16
; SI-NEXT: s_or_b32 s7, s7, s41
; SI-NEXT: s_or_b32 s5, s5, s40
-; SI-NEXT: v_readlane_b32 s30, v26, 7
+; SI-NEXT: v_readlane_b32 s30, v26, 5
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -27748,9 +28202,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v23, s7
; SI-NEXT: v_mov_b32_e32 v24, s4
; SI-NEXT: v_mov_b32_e32 v25, s5
-; SI-NEXT: v_readlane_b32 s31, v26, 8
-; SI-NEXT: v_readlane_b32 s48, v26, 6
-; SI-NEXT: v_readlane_b32 s39, v26, 5
+; SI-NEXT: v_readlane_b32 s31, v26, 6
; SI-NEXT: v_readlane_b32 s38, v26, 4
; SI-NEXT: v_readlane_b32 s37, v26, 3
; SI-NEXT: v_readlane_b32 s36, v26, 2
@@ -27761,34 +28213,6 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v13i64_to_v52f16_scalar:
; VI: ; %bb.0:
@@ -27807,7 +28231,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s40, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s41, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s6, 16
; VI-NEXT: s_lshr_b32 s43, s7, 16
@@ -27835,8 +28259,42 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -27889,7 +28347,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s18, 16
; VI-NEXT: s_lshr_b32 s90, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -27995,34 +28453,6 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v24, s7
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v13i64_to_v52f16_scalar:
; GFX9: ; %bb.0:
@@ -28041,7 +28471,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s40, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s41, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s6, 16
; GFX9-NEXT: s_lshr_b32 s43, s7, 16
@@ -28069,8 +28499,42 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -28123,7 +28587,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s89, s18, 16
; GFX9-NEXT: s_lshr_b32 s90, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s91
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s90
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s89
@@ -28177,34 +28641,6 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v24, s7
; GFX9-NEXT: v_mov_b32_e32 v25, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v13i64_to_v52f16_scalar:
; GFX11: ; %bb.0:
@@ -28220,7 +28656,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s11, v0
; GFX11-NEXT: s_cmp_lg_u32 s12, 0
; GFX11-NEXT: s_mov_b32 s78, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s12, s4, 16
; GFX11-NEXT: s_lshr_b32 s13, s5, 16
@@ -28248,9 +28684,42 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s78, -1
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -28303,7 +28772,7 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s75, s2, 16
; GFX11-NEXT: s_lshr_b32 s76, s1, 16
; GFX11-NEXT: s_lshr_b32 s77, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s77
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s76
@@ -28345,34 +28814,6 @@ define inreg <52 x half> @bitcast_v13i64_to_v52f16_scalar(<13 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v22, s7 :: v_dual_mov_b32 v23, s6
; GFX11-NEXT: v_dual_mov_b32 v24, s5 :: v_dual_mov_b32 v25, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -29692,10 +30133,8 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s71, 21
; SI-NEXT: v_writelane_b32 v32, s80, 22
; SI-NEXT: v_writelane_b32 v32, s81, 23
-; SI-NEXT: v_writelane_b32 v32, s82, 24
-; SI-NEXT: v_writelane_b32 v32, s83, 25
-; SI-NEXT: v_writelane_b32 v32, s30, 26
-; SI-NEXT: v_writelane_b32 v32, s31, 27
+; SI-NEXT: v_writelane_b32 v32, s30, 24
+; SI-NEXT: v_writelane_b32 v32, s31, 25
; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: v_readfirstlane_b32 s8, v10
; SI-NEXT: v_readfirstlane_b32 s10, v9
@@ -29710,18 +30149,18 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s94, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s93, s28, 16
-; SI-NEXT: s_lshr_b32 s30, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -29736,43 +30175,43 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s95, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -29816,11 +30255,20 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s83
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s81
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s80
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -29832,7 +30280,7 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -29840,11 +30288,11 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s69
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -29855,7 +30303,7 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s68
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -29864,11 +30312,11 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s34
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -29879,7 +30327,7 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -29888,11 +30336,11 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s30
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_hi
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -29903,7 +30351,7 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -30026,11 +30474,8 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v24, v25, v24
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v26
; SI-NEXT: v_or_b32_e32 v25, v27, v25
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -30063,11 +30508,9 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 26
-; SI-NEXT: v_readlane_b32 s31, v32, 27
-; SI-NEXT: v_readlane_b32 s83, v32, 25
-; SI-NEXT: v_readlane_b32 s82, v32, 24
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 24
+; SI-NEXT: v_readlane_b32 s31, v32, 25
; SI-NEXT: v_readlane_b32 s81, v32, 23
; SI-NEXT: v_readlane_b32 s80, v32, 22
; SI-NEXT: v_readlane_b32 s71, v32, 21
@@ -30132,10 +30575,8 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s83, 25
; VI-NEXT: v_writelane_b32 v32, s84, 26
; VI-NEXT: v_writelane_b32 v32, s85, 27
-; VI-NEXT: v_writelane_b32 v32, s86, 28
-; VI-NEXT: v_writelane_b32 v32, s87, 29
-; VI-NEXT: v_writelane_b32 v32, s30, 30
-; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_writelane_b32 v32, s30, 28
+; VI-NEXT: v_writelane_b32 v32, s31, 29
; VI-NEXT: v_readfirstlane_b32 s6, v11
; VI-NEXT: v_readfirstlane_b32 s8, v10
; VI-NEXT: v_readfirstlane_b32 s10, v9
@@ -30145,23 +30586,23 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s89, v3
-; VI-NEXT: v_readfirstlane_b32 s30, v2
-; VI-NEXT: v_readfirstlane_b32 s35, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; VI-NEXT: v_readfirstlane_b32 s31, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s74, s29, 16
; VI-NEXT: s_lshr_b32 s77, s28, 16
; VI-NEXT: s_lshr_b32 s88, s27, 16
; VI-NEXT: s_lshr_b32 s90, s26, 16
-; VI-NEXT: s_lshr_b32 s31, s25, 16
-; VI-NEXT: s_lshr_b32 s68, s24, 16
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s25, 16
+; VI-NEXT: s_lshr_b32 s34, s24, 16
+; VI-NEXT: s_lshr_b32 s68, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -30171,42 +30612,42 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s76, s75, 16
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s91, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -30220,14 +30661,14 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -30256,46 +30697,55 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v25, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s87
-; VI-NEXT: v_mov_b32_e32 v2, s86
+; VI-NEXT: v_mov_b32_e32 v0, s85
+; VI-NEXT: v_mov_b32_e32 v2, s84
; VI-NEXT: v_add_f16_sdwa v0, v0, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v25
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v25
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_mov_b32_e32 v2, s83
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v25
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s84
+; VI-NEXT: v_mov_b32_e32 v3, s82
; VI-NEXT: v_add_f16_sdwa v3, v3, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v25
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s83
+; VI-NEXT: v_mov_b32_e32 v4, s81
; VI-NEXT: v_add_f16_sdwa v4, v4, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v25
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s82
+; VI-NEXT: v_mov_b32_e32 v5, s80
; VI-NEXT: v_add_f16_sdwa v5, v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v25
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s81
+; VI-NEXT: v_mov_b32_e32 v6, s71
; VI-NEXT: v_add_f16_sdwa v6, v6, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v25
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s70
+; VI-NEXT: v_mov_b32_e32 v7, s68
; VI-NEXT: v_add_f16_sdwa v7, v7, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v25
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s68
+; VI-NEXT: v_mov_b32_e32 v8, s34
; VI-NEXT: v_add_f16_sdwa v8, v8, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v25
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s31
+; VI-NEXT: v_mov_b32_e32 v9, vcc_hi
; VI-NEXT: v_add_f16_sdwa v9, v9, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v25
; VI-NEXT: v_or_b32_e32 v9, v10, v9
@@ -30315,17 +30765,17 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v13, v13, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v25
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s80
+; VI-NEXT: v_mov_b32_e32 v14, s70
; VI-NEXT: v_add_f16_sdwa v14, v14, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s71, v25
+; VI-NEXT: v_add_f16_e32 v15, s69, v25
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s69
+; VI-NEXT: v_mov_b32_e32 v15, s35
; VI-NEXT: v_add_f16_sdwa v15, v15, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s35, v25
+; VI-NEXT: v_add_f16_e32 v16, s31, v25
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_mov_b32_e32 v16, s30
; VI-NEXT: v_add_f16_sdwa v16, v16, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s30, v25
+; VI-NEXT: v_add_f16_e32 v17, vcc_lo, v25
; VI-NEXT: v_or_b32_e32 v16, v17, v16
; VI-NEXT: v_mov_b32_e32 v17, s91
; VI-NEXT: v_add_f16_sdwa v17, v17, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -30363,11 +30813,8 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v26, v26, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s6, v25
; VI-NEXT: v_or_b32_e32 v25, v25, v26
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -30400,11 +30847,9 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s31, v32, 31
-; VI-NEXT: v_readlane_b32 s87, v32, 29
-; VI-NEXT: v_readlane_b32 s86, v32, 28
+; VI-NEXT: .LBB47_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 28
+; VI-NEXT: v_readlane_b32 s31, v32, 29
; VI-NEXT: v_readlane_b32 s85, v32, 27
; VI-NEXT: v_readlane_b32 s84, v32, 26
; VI-NEXT: v_readlane_b32 s83, v32, 25
@@ -30523,10 +30968,18 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v25, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v25 op_sel_hi:[1,0]
@@ -30554,10 +31007,8 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v23, s59, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v24, s60, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v25, s61, v25 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -30590,7 +31041,7 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -30675,11 +31126,16 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -30707,8 +31163,6 @@ define inreg <13 x i64> @bitcast_v52f16_to_v13i64_scalar(<52 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v24, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v25, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -31455,10 +31909,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_writelane_b32 v45, s36, 2
; SI-NEXT: v_writelane_b32 v45, s37, 3
; SI-NEXT: v_writelane_b32 v45, s38, 4
-; SI-NEXT: v_writelane_b32 v45, s39, 5
-; SI-NEXT: v_writelane_b32 v45, s48, 6
-; SI-NEXT: v_writelane_b32 v45, s30, 7
-; SI-NEXT: v_writelane_b32 v45, s31, 8
+; SI-NEXT: v_writelane_b32 v45, s30, 5
+; SI-NEXT: v_writelane_b32 v45, s31, 6
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: v_readfirstlane_b32 s13, v11
; SI-NEXT: v_readfirstlane_b32 s12, v10
@@ -31473,17 +31925,17 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s48, s13, 16
-; SI-NEXT: s_lshr_b32 s39, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s11, 16
-; SI-NEXT: s_lshr_b32 s37, s9, 16
-; SI-NEXT: s_lshr_b32 s36, s7, 16
-; SI-NEXT: s_lshr_b32 s35, s5, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s31, s27, 16
-; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s38, s13, 16
+; SI-NEXT: s_lshr_b32 s37, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s11, 16
+; SI-NEXT: s_lshr_b32 s35, s9, 16
+; SI-NEXT: s_lshr_b32 s34, s7, 16
+; SI-NEXT: s_lshr_b32 s31, s5, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s25, 16
; SI-NEXT: s_lshr_b32 s95, s23, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s93, s19, 16
@@ -31501,8 +31953,42 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[14:15], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
@@ -31547,36 +32033,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -31603,16 +32061,16 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s14
; SI-NEXT: v_mov_b32_e32 v25, s13
; SI-NEXT: v_mov_b32_e32 v24, s12
-; SI-NEXT: v_mov_b32_e32 v48, s48
-; SI-NEXT: v_mov_b32_e32 v49, s39
-; SI-NEXT: v_mov_b32_e32 v50, s38
-; SI-NEXT: v_mov_b32_e32 v51, s37
-; SI-NEXT: v_mov_b32_e32 v52, s36
-; SI-NEXT: v_mov_b32_e32 v53, s35
-; SI-NEXT: v_mov_b32_e32 v54, s34
-; SI-NEXT: v_mov_b32_e32 v55, s31
+; SI-NEXT: v_mov_b32_e32 v48, s38
+; SI-NEXT: v_mov_b32_e32 v49, s37
+; SI-NEXT: v_mov_b32_e32 v50, s36
+; SI-NEXT: v_mov_b32_e32 v51, s35
+; SI-NEXT: v_mov_b32_e32 v52, s34
+; SI-NEXT: v_mov_b32_e32 v53, s31
+; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v55, vcc_hi
; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v40, s30
+; SI-NEXT: v_mov_b32_e32 v40, vcc_lo
; SI-NEXT: s_waitcnt expcnt(3)
; SI-NEXT: v_mov_b32_e32 v41, s95
; SI-NEXT: s_waitcnt expcnt(2)
@@ -31634,7 +32092,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v28, s44
; SI-NEXT: v_mov_b32_e32 v27, s42
; SI-NEXT: v_mov_b32_e32 v26, s40
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -31705,7 +32163,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v24, v24, v26
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v48
-; SI-NEXT: v_readlane_b32 s30, v45, 7
+; SI-NEXT: v_readlane_b32 s30, v45, 5
; SI-NEXT: v_or_b32_e32 v1, v1, v38
; SI-NEXT: v_or_b32_e32 v3, v3, v37
; SI-NEXT: v_or_b32_e32 v5, v5, v36
@@ -31719,9 +32177,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v21, v21, v28
; SI-NEXT: v_or_b32_e32 v23, v23, v27
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: v_readlane_b32 s31, v45, 8
-; SI-NEXT: v_readlane_b32 s48, v45, 6
-; SI-NEXT: v_readlane_b32 s39, v45, 5
+; SI-NEXT: v_readlane_b32 s31, v45, 6
; SI-NEXT: v_readlane_b32 s38, v45, 4
; SI-NEXT: v_readlane_b32 s37, v45, 3
; SI-NEXT: v_readlane_b32 s36, v45, 2
@@ -31754,7 +32210,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s9, 16
; VI-NEXT: s_lshr_b32 s63, s8, 16
@@ -31782,8 +32238,42 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s90, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[40:41], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[40:41], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[40:41], s[40:41], exec
+; VI-NEXT: s_cselect_b32 s40, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s40, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[12:13], 1.0
@@ -31823,36 +32313,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v40, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -31905,7 +32367,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v33, s44
; VI-NEXT: v_mov_b32_e32 v31, s43
; VI-NEXT: v_mov_b32_e32 v28, s42
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
; VI-NEXT: v_or_b32_sdwa v0, v0, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v43
@@ -31986,7 +32448,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s9, 16
; GFX9-NEXT: s_lshr_b32 s63, s8, 16
@@ -32014,8 +32476,42 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s90, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[40:41], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[40:41], -1
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr43
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr42
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[40:41], s[40:41], exec
+; GFX9-NEXT: s_cselect_b32 s40, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s40, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[20:21], s[12:13], 1.0
@@ -32055,36 +32551,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v43, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v40, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr43
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -32137,7 +32605,7 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v33, s44
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
@@ -32210,12 +32678,12 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s9, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s8, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s59, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s4, 16
@@ -32239,9 +32707,42 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s77, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
@@ -32281,36 +32782,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -32336,8 +32809,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s61 :: v_dual_mov_b32 v37, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s58 :: v_dual_mov_b32 v35, s40
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v66, s56 :: v_dual_mov_b32 v33, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s14 :: v_dual_mov_b32 v28, s13
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s13 :: v_dual_mov_b32 v28, s12
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v64.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v67.l
@@ -32380,12 +32853,12 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s9, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s8, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s59, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s4, 16
@@ -32409,50 +32882,9 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s77, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[8:9], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[6:7], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[4:5], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[16:17], s[10:11], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[18:19], s[28:29], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[11:12], s[26:27], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[13:14], s[24:25], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[22:23], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[6:7], s[20:21], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[4:5], s[0:1], 1.0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 16, v22
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 16, v21
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v32, 16, v24
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v23
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v26
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v20, 16, v25
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v17
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v16
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v19
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v18
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v11
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v48, 16, v14
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v13
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v28
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v27
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v9
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v64, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, -1
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
@@ -32476,11 +32908,57 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[8:9], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[6:7], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[4:5], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[16:17], s[10:11], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[18:19], s[28:29], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[11:12], s[26:27], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[13:14], s[24:25], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[22:23], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[6:7], s[20:21], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[8:9], s[18:19], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[2:3], s[2:3], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[4:5], s[0:1], 1.0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 16, v21
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v32, 16, v24
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v23
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v33, 16, v26
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v20, 16, v25
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v34, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v19
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v11
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v48, 16, v14
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v28
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v27
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v6
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v9
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v64, 16, v4
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v27, s22
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v13, s24
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v11, s26
@@ -32506,8 +32984,8 @@ define inreg <52 x i16> @bitcast_v13f64_to_v52i16_scalar(<13 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, s45 :: v_dual_mov_b32 v39, s42
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, s43 :: v_dual_mov_b32 v37, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s40 :: v_dual_mov_b32 v33, s15
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s14 :: v_dual_mov_b32 v31, s13
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s13 :: v_dual_mov_b32 v31, s12
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v68, 0xffff, v0
@@ -33759,10 +34237,8 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v26, s71, 21
; SI-NEXT: v_writelane_b32 v26, s80, 22
; SI-NEXT: v_writelane_b32 v26, s81, 23
-; SI-NEXT: v_writelane_b32 v26, s82, 24
-; SI-NEXT: v_writelane_b32 v26, s83, 25
-; SI-NEXT: v_writelane_b32 v26, s30, 26
-; SI-NEXT: v_writelane_b32 v26, s31, 27
+; SI-NEXT: v_writelane_b32 v26, s30, 24
+; SI-NEXT: v_writelane_b32 v26, s31, 25
; SI-NEXT: v_readfirstlane_b32 s7, v11
; SI-NEXT: v_readfirstlane_b32 s9, v10
; SI-NEXT: v_readfirstlane_b32 s11, v9
@@ -33773,22 +34249,22 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s78, v4
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s93, v2
-; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: v_readfirstlane_b32 s35, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; SI-NEXT: v_readfirstlane_b32 s31, v0
; SI-NEXT: s_lshr_b32 s76, s29, 16
; SI-NEXT: s_lshr_b32 s79, s28, 16
; SI-NEXT: s_lshr_b32 s89, s27, 16
; SI-NEXT: s_lshr_b32 s91, s26, 16
; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s31, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -33799,38 +34275,38 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s77, s78, 16
; SI-NEXT: s_lshr_b32 s88, s90, 16
; SI-NEXT: s_lshr_b32 s92, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -33847,10 +34323,10 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -33883,51 +34359,60 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
@@ -33954,14 +34439,14 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s76, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -34015,8 +34500,8 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s61, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v26, 26
+; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v26, 24
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -34043,9 +34528,7 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v23, s59
; SI-NEXT: v_mov_b32_e32 v24, s60
; SI-NEXT: v_mov_b32_e32 v25, s61
-; SI-NEXT: v_readlane_b32 s31, v26, 27
-; SI-NEXT: v_readlane_b32 s83, v26, 25
-; SI-NEXT: v_readlane_b32 s82, v26, 24
+; SI-NEXT: v_readlane_b32 s31, v26, 25
; SI-NEXT: v_readlane_b32 s81, v26, 23
; SI-NEXT: v_readlane_b32 s80, v26, 22
; SI-NEXT: v_readlane_b32 s71, v26, 21
@@ -34075,9 +34558,6 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v52i16_to_v13f64_scalar:
; VI: ; %bb.0:
@@ -34113,10 +34593,8 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v26, s83, 25
; VI-NEXT: v_writelane_b32 v26, s84, 26
; VI-NEXT: v_writelane_b32 v26, s85, 27
-; VI-NEXT: v_writelane_b32 v26, s86, 28
-; VI-NEXT: v_writelane_b32 v26, s87, 29
-; VI-NEXT: v_writelane_b32 v26, s30, 30
-; VI-NEXT: v_writelane_b32 v26, s31, 31
+; VI-NEXT: v_writelane_b32 v26, s30, 28
+; VI-NEXT: v_writelane_b32 v26, s31, 29
; VI-NEXT: v_readfirstlane_b32 s7, v11
; VI-NEXT: v_readfirstlane_b32 s9, v10
; VI-NEXT: v_readfirstlane_b32 s11, v9
@@ -34126,23 +34604,23 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s90, v3
-; VI-NEXT: v_readfirstlane_b32 s31, v2
-; VI-NEXT: v_readfirstlane_b32 s68, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; VI-NEXT: v_readfirstlane_b32 s34, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s76, s29, 16
; VI-NEXT: s_lshr_b32 s79, s28, 16
; VI-NEXT: s_lshr_b32 s89, s27, 16
; VI-NEXT: s_lshr_b32 s91, s26, 16
-; VI-NEXT: s_lshr_b32 s34, s25, 16
-; VI-NEXT: s_lshr_b32 s69, s24, 16
-; VI-NEXT: s_lshr_b32 s80, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 s30, s25, 16
+; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s70, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s8, s9, 16
; VI-NEXT: s_lshr_b32 s10, s11, 16
@@ -34152,42 +34630,42 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_lshr_b32 s74, s75, 16
; VI-NEXT: s_lshr_b32 s77, s78, 16
; VI-NEXT: s_lshr_b32 s88, s90, 16
-; VI-NEXT: s_lshr_b32 s30, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s31, s34, 16
+; VI-NEXT: s_lshr_b32 s68, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -34201,14 +34679,14 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s90
; VI-NEXT: s_lshl_b32 s5, s88, 16
@@ -34237,56 +34715,65 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s7
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s37, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -34308,20 +34795,20 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s90, s90, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -34369,8 +34856,8 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s6, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s61, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
-; VI-NEXT: v_readlane_b32 s30, v26, 30
+; VI-NEXT: .LBB51_5: ; %end
+; VI-NEXT: v_readlane_b32 s30, v26, 28
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -34397,9 +34884,7 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: v_mov_b32_e32 v23, s59
; VI-NEXT: v_mov_b32_e32 v24, s60
; VI-NEXT: v_mov_b32_e32 v25, s61
-; VI-NEXT: v_readlane_b32 s31, v26, 31
-; VI-NEXT: v_readlane_b32 s87, v26, 29
-; VI-NEXT: v_readlane_b32 s86, v26, 28
+; VI-NEXT: v_readlane_b32 s31, v26, 29
; VI-NEXT: v_readlane_b32 s85, v26, 27
; VI-NEXT: v_readlane_b32 s84, v26, 26
; VI-NEXT: v_readlane_b32 s83, v26, 25
@@ -34433,9 +34918,6 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v52i16_to_v13f64_scalar:
; GFX9: ; %bb.0:
@@ -34521,10 +35003,18 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -34551,10 +35041,8 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v24, s60, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -34587,7 +35075,7 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -34672,11 +35160,16 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -34704,8 +35197,6 @@ define inreg <13 x double> @bitcast_v52i16_to_v13f64_scalar(<52 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v24, s24, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v25, s25, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -35452,10 +35943,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_writelane_b32 v45, s36, 2
; SI-NEXT: v_writelane_b32 v45, s37, 3
; SI-NEXT: v_writelane_b32 v45, s38, 4
-; SI-NEXT: v_writelane_b32 v45, s39, 5
-; SI-NEXT: v_writelane_b32 v45, s48, 6
-; SI-NEXT: v_writelane_b32 v45, s30, 7
-; SI-NEXT: v_writelane_b32 v45, s31, 8
+; SI-NEXT: v_writelane_b32 v45, s30, 5
+; SI-NEXT: v_writelane_b32 v45, s31, 6
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: v_readfirstlane_b32 s13, v11
; SI-NEXT: v_readfirstlane_b32 s12, v10
@@ -35470,17 +35959,17 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s48, s13, 16
-; SI-NEXT: s_lshr_b32 s39, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s11, 16
-; SI-NEXT: s_lshr_b32 s37, s9, 16
-; SI-NEXT: s_lshr_b32 s36, s7, 16
-; SI-NEXT: s_lshr_b32 s35, s5, 16
-; SI-NEXT: s_lshr_b32 s34, s29, 16
-; SI-NEXT: s_lshr_b32 s31, s27, 16
-; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s38, s13, 16
+; SI-NEXT: s_lshr_b32 s37, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s11, 16
+; SI-NEXT: s_lshr_b32 s35, s9, 16
+; SI-NEXT: s_lshr_b32 s34, s7, 16
+; SI-NEXT: s_lshr_b32 s31, s5, 16
+; SI-NEXT: s_lshr_b32 s30, s29, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s25, 16
; SI-NEXT: s_lshr_b32 s95, s23, 16
; SI-NEXT: s_lshr_b32 s94, s21, 16
; SI-NEXT: s_lshr_b32 s93, s19, 16
@@ -35498,8 +35987,42 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[76:77], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[90:91], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[90:91], -1
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr93
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr95
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[90:91], s[90:91], exec
+; SI-NEXT: s_cselect_b32 s41, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s41, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[14:15], 1.0
; SI-NEXT: v_add_f64 v[20:21], s[10:11], 1.0
@@ -35544,36 +36067,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr93
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr95
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -35600,16 +36095,16 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v22, s14
; SI-NEXT: v_mov_b32_e32 v25, s13
; SI-NEXT: v_mov_b32_e32 v24, s12
-; SI-NEXT: v_mov_b32_e32 v48, s48
-; SI-NEXT: v_mov_b32_e32 v49, s39
-; SI-NEXT: v_mov_b32_e32 v50, s38
-; SI-NEXT: v_mov_b32_e32 v51, s37
-; SI-NEXT: v_mov_b32_e32 v52, s36
-; SI-NEXT: v_mov_b32_e32 v53, s35
-; SI-NEXT: v_mov_b32_e32 v54, s34
-; SI-NEXT: v_mov_b32_e32 v55, s31
+; SI-NEXT: v_mov_b32_e32 v48, s38
+; SI-NEXT: v_mov_b32_e32 v49, s37
+; SI-NEXT: v_mov_b32_e32 v50, s36
+; SI-NEXT: v_mov_b32_e32 v51, s35
+; SI-NEXT: v_mov_b32_e32 v52, s34
+; SI-NEXT: v_mov_b32_e32 v53, s31
+; SI-NEXT: v_mov_b32_e32 v54, s30
+; SI-NEXT: v_mov_b32_e32 v55, vcc_hi
; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v40, s30
+; SI-NEXT: v_mov_b32_e32 v40, vcc_lo
; SI-NEXT: s_waitcnt expcnt(3)
; SI-NEXT: v_mov_b32_e32 v41, s95
; SI-NEXT: s_waitcnt expcnt(2)
@@ -35631,7 +36126,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v28, s44
; SI-NEXT: v_mov_b32_e32 v27, s42
; SI-NEXT: v_mov_b32_e32 v26, s40
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v37, 16, v37
@@ -35702,7 +36197,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v24, v24, v26
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v48
-; SI-NEXT: v_readlane_b32 s30, v45, 7
+; SI-NEXT: v_readlane_b32 s30, v45, 5
; SI-NEXT: v_or_b32_e32 v1, v1, v38
; SI-NEXT: v_or_b32_e32 v3, v3, v37
; SI-NEXT: v_or_b32_e32 v5, v5, v36
@@ -35716,9 +36211,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v21, v21, v28
; SI-NEXT: v_or_b32_e32 v23, v23, v27
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: v_readlane_b32 s31, v45, 8
-; SI-NEXT: v_readlane_b32 s48, v45, 6
-; SI-NEXT: v_readlane_b32 s39, v45, 5
+; SI-NEXT: v_readlane_b32 s31, v45, 6
; SI-NEXT: v_readlane_b32 s38, v45, 4
; SI-NEXT: v_readlane_b32 s37, v45, 3
; SI-NEXT: v_readlane_b32 s36, v45, 2
@@ -35751,7 +36244,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s42, s9, 16
; VI-NEXT: s_lshr_b32 s63, s8, 16
@@ -35779,8 +36272,42 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; VI-NEXT: s_lshr_b32 s90, s18, 16
; VI-NEXT: s_lshr_b32 s62, s17, 16
; VI-NEXT: s_lshr_b32 s91, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[40:41], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[40:41], -1
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr43
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr42
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[40:41], s[40:41], exec
+; VI-NEXT: s_cselect_b32 s40, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s40, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[20:21], s[12:13], 1.0
@@ -35820,36 +36347,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v43, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v40, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr43
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr42
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -35902,7 +36401,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v33, s44
; VI-NEXT: v_mov_b32_e32 v31, s43
; VI-NEXT: v_mov_b32_e32 v28, s42
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
; VI-NEXT: v_or_b32_sdwa v0, v0, v40 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v43
@@ -35983,7 +36482,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s42, s9, 16
; GFX9-NEXT: s_lshr_b32 s63, s8, 16
@@ -36011,49 +36510,10 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s90, s18, 16
; GFX9-NEXT: s_lshr_b32 s62, s17, 16
; GFX9-NEXT: s_lshr_b32 s91, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
-; GFX9-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
-; GFX9-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
-; GFX9-NEXT: v_add_f64 v[20:21], s[12:13], 1.0
-; GFX9-NEXT: v_add_f64 v[18:19], s[14:15], 1.0
-; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
-; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
-; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
-; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
-; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
-; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
-; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
-; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v25
-; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v24
-; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v23
-; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v22
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v21
-; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v19
-; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v18
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v17
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v16
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v15
-; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v13
-; GFX9-NEXT: v_lshrrev_b32_e32 v36, 16, v12
-; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v11
-; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v9
-; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v52, 16, v4
-; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v43, 16, v1
-; GFX9-NEXT: v_lshrrev_b32_e32 v40, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
+; GFX9-NEXT: s_mov_b64 s[40:41], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[40:41], -1
; GFX9-NEXT: ; implicit-def: $sgpr91
; GFX9-NEXT: ; implicit-def: $sgpr62
; GFX9-NEXT: ; implicit-def: $sgpr90
@@ -36080,8 +36540,53 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: ; implicit-def: $sgpr43
; GFX9-NEXT: ; implicit-def: $sgpr63
; GFX9-NEXT: ; implicit-def: $sgpr42
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[40:41], s[40:41], exec
+; GFX9-NEXT: s_cselect_b32 s40, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s40, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
+; GFX9-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
+; GFX9-NEXT: v_add_f64 v[20:21], s[12:13], 1.0
+; GFX9-NEXT: v_add_f64 v[18:19], s[14:15], 1.0
+; GFX9-NEXT: v_add_f64 v[16:17], s[6:7], 1.0
+; GFX9-NEXT: v_add_f64 v[14:15], s[4:5], 1.0
+; GFX9-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
+; GFX9-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
+; GFX9-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; GFX9-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; GFX9-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; GFX9-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; GFX9-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v25
+; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v24
+; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v23
+; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v22
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v21
+; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v20
+; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v19
+; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v18
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v17
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 16, v15
+; GFX9-NEXT: v_lshrrev_b32_e32 v34, 16, v14
+; GFX9-NEXT: v_lshrrev_b32_e32 v49, 16, v13
+; GFX9-NEXT: v_lshrrev_b32_e32 v36, 16, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v11
+; GFX9-NEXT: v_lshrrev_b32_e32 v38, 16, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v48, 16, v8
+; GFX9-NEXT: v_lshrrev_b32_e32 v55, 16, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v52, 16, v4
+; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v54, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v43, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v40, 16, v0
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -36134,7 +36639,7 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v33, s44
; GFX9-NEXT: v_mov_b32_e32 v31, s43
; GFX9-NEXT: v_mov_b32_e32 v28, s42
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
@@ -36207,12 +36712,12 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s12, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s9, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s58, s8, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s7, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s59, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s4, 16
@@ -36236,9 +36741,42 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s77, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[20:21], s[4:5], 1.0
@@ -36278,36 +36816,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -36333,8 +36843,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s61 :: v_dual_mov_b32 v37, s41
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s58 :: v_dual_mov_b32 v35, s40
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v66, s56 :: v_dual_mov_b32 v33, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s14 :: v_dual_mov_b32 v28, s13
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s13 :: v_dual_mov_b32 v28, s12
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v64.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v67.l
@@ -36377,12 +36887,12 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s12, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s12, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s9, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s12, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s58, s8, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s7, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s13, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s59, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s4, 16
@@ -36406,9 +36916,42 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s77, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s57, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s78, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s12
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s14, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[21:22], s[8:9], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[6:7], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[4:5], 1.0
@@ -36448,36 +36991,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v64, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v27, s22
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v13, s24
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v11, s26
@@ -36503,8 +37018,8 @@ define inreg <52 x half> @bitcast_v13f64_to_v52f16_scalar(<13 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, s45 :: v_dual_mov_b32 v39, s42
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, s43 :: v_dual_mov_b32 v37, s41
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s40 :: v_dual_mov_b32 v33, s15
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s14 :: v_dual_mov_b32 v31, s13
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v32, s13 :: v_dual_mov_b32 v31, s12
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v68, 0xffff, v0
@@ -37877,10 +38392,8 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_writelane_b32 v32, s71, 21
; SI-NEXT: v_writelane_b32 v32, s80, 22
; SI-NEXT: v_writelane_b32 v32, s81, 23
-; SI-NEXT: v_writelane_b32 v32, s82, 24
-; SI-NEXT: v_writelane_b32 v32, s83, 25
-; SI-NEXT: v_writelane_b32 v32, s30, 26
-; SI-NEXT: v_writelane_b32 v32, s31, 27
+; SI-NEXT: v_writelane_b32 v32, s30, 24
+; SI-NEXT: v_writelane_b32 v32, s31, 25
; SI-NEXT: v_readfirstlane_b32 s6, v11
; SI-NEXT: v_readfirstlane_b32 s8, v10
; SI-NEXT: v_readfirstlane_b32 s10, v9
@@ -37895,18 +38408,18 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_readfirstlane_b32 s94, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s93, s28, 16
-; SI-NEXT: s_lshr_b32 s30, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s24, 16
-; SI-NEXT: s_lshr_b32 s68, s23, 16
-; SI-NEXT: s_lshr_b32 s69, s22, 16
-; SI-NEXT: s_lshr_b32 s70, s21, 16
-; SI-NEXT: s_lshr_b32 s71, s20, 16
-; SI-NEXT: s_lshr_b32 s80, s19, 16
-; SI-NEXT: s_lshr_b32 s81, s18, 16
-; SI-NEXT: s_lshr_b32 s82, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s27, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s24, 16
+; SI-NEXT: s_lshr_b32 s34, s23, 16
+; SI-NEXT: s_lshr_b32 s35, s22, 16
+; SI-NEXT: s_lshr_b32 s68, s21, 16
+; SI-NEXT: s_lshr_b32 s69, s20, 16
+; SI-NEXT: s_lshr_b32 s70, s19, 16
+; SI-NEXT: s_lshr_b32 s71, s18, 16
+; SI-NEXT: s_lshr_b32 s80, s17, 16
+; SI-NEXT: s_lshr_b32 s81, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -37921,43 +38434,43 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: s_lshr_b32 s95, s94, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -38001,11 +38514,20 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s61, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s83
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s81
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s80
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -38017,7 +38539,7 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v4, s18
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s71
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -38025,11 +38547,11 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v3, s70
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: v_cvt_f32_f16_e32 v4, s19
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s69
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
@@ -38040,7 +38562,7 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s68
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
@@ -38049,11 +38571,11 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s35
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v7, v5
; SI-NEXT: v_cvt_f32_f16_e32 v7, s22
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s34
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
@@ -38064,7 +38586,7 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_or_b32_e32 v6, v7, v6
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s31
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
@@ -38073,11 +38595,11 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_or_b32_e32 v7, v9, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s30
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: v_cvt_f32_f16_e32 v10, s25
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_hi
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
@@ -38088,7 +38610,7 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v11, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
@@ -38211,11 +38733,8 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v24, v25, v24
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v26
; SI-NEXT: v_or_b32_e32 v25, v27, v25
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -38248,11 +38767,9 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 26
-; SI-NEXT: v_readlane_b32 s31, v32, 27
-; SI-NEXT: v_readlane_b32 s83, v32, 25
-; SI-NEXT: v_readlane_b32 s82, v32, 24
+; SI-NEXT: .LBB55_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 24
+; SI-NEXT: v_readlane_b32 s31, v32, 25
; SI-NEXT: v_readlane_b32 s81, v32, 23
; SI-NEXT: v_readlane_b32 s80, v32, 22
; SI-NEXT: v_readlane_b32 s71, v32, 21
@@ -38317,10 +38834,8 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: v_writelane_b32 v32, s83, 25
; VI-NEXT: v_writelane_b32 v32, s84, 26
; VI-NEXT: v_writelane_b32 v32, s85, 27
-; VI-NEXT: v_writelane_b32 v32, s86, 28
-; VI-NEXT: v_writelane_b32 v32, s87, 29
-; VI-NEXT: v_writelane_b32 v32, s30, 30
-; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_writelane_b32 v32, s30, 28
+; VI-NEXT: v_writelane_b32 v32, s31, 29
; VI-NEXT: v_readfirstlane_b32 s6, v11
; VI-NEXT: v_readfirstlane_b32 s8, v10
; VI-NEXT: v_readfirstlane_b32 s10, v9
@@ -38330,23 +38845,23 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: v_readfirstlane_b32 s75, v5
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s89, v3
-; VI-NEXT: v_readfirstlane_b32 s30, v2
-; VI-NEXT: v_readfirstlane_b32 s35, v1
-; VI-NEXT: v_readfirstlane_b32 s71, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; VI-NEXT: v_readfirstlane_b32 s31, v1
+; VI-NEXT: v_readfirstlane_b32 s69, v0
; VI-NEXT: s_lshr_b32 s74, s29, 16
; VI-NEXT: s_lshr_b32 s77, s28, 16
; VI-NEXT: s_lshr_b32 s88, s27, 16
; VI-NEXT: s_lshr_b32 s90, s26, 16
-; VI-NEXT: s_lshr_b32 s31, s25, 16
-; VI-NEXT: s_lshr_b32 s68, s24, 16
-; VI-NEXT: s_lshr_b32 s70, s23, 16
-; VI-NEXT: s_lshr_b32 s81, s22, 16
-; VI-NEXT: s_lshr_b32 s82, s21, 16
-; VI-NEXT: s_lshr_b32 s83, s20, 16
-; VI-NEXT: s_lshr_b32 s84, s19, 16
-; VI-NEXT: s_lshr_b32 s85, s18, 16
-; VI-NEXT: s_lshr_b32 s86, s17, 16
-; VI-NEXT: s_lshr_b32 s87, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s25, 16
+; VI-NEXT: s_lshr_b32 s34, s24, 16
+; VI-NEXT: s_lshr_b32 s68, s23, 16
+; VI-NEXT: s_lshr_b32 s71, s22, 16
+; VI-NEXT: s_lshr_b32 s80, s21, 16
+; VI-NEXT: s_lshr_b32 s81, s20, 16
+; VI-NEXT: s_lshr_b32 s82, s19, 16
+; VI-NEXT: s_lshr_b32 s83, s18, 16
+; VI-NEXT: s_lshr_b32 s84, s17, 16
+; VI-NEXT: s_lshr_b32 s85, s16, 16
; VI-NEXT: s_lshr_b32 s7, s6, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -38356,42 +38871,42 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: s_lshr_b32 s76, s75, 16
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s91, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s86, 16
+; VI-NEXT: s_lshl_b32 s41, s84, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -38405,14 +38920,14 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s52, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
; VI-NEXT: s_lshl_b32 s5, s91, 16
@@ -38441,46 +38956,55 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v25, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s87
-; VI-NEXT: v_mov_b32_e32 v2, s86
+; VI-NEXT: v_mov_b32_e32 v0, s85
+; VI-NEXT: v_mov_b32_e32 v2, s84
; VI-NEXT: v_add_f16_sdwa v0, v0, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v25
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v25
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s85
+; VI-NEXT: v_mov_b32_e32 v2, s83
; VI-NEXT: v_add_f16_sdwa v2, v2, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v25
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s84
+; VI-NEXT: v_mov_b32_e32 v3, s82
; VI-NEXT: v_add_f16_sdwa v3, v3, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v25
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s83
+; VI-NEXT: v_mov_b32_e32 v4, s81
; VI-NEXT: v_add_f16_sdwa v4, v4, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v25
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s82
+; VI-NEXT: v_mov_b32_e32 v5, s80
; VI-NEXT: v_add_f16_sdwa v5, v5, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v25
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s81
+; VI-NEXT: v_mov_b32_e32 v6, s71
; VI-NEXT: v_add_f16_sdwa v6, v6, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v25
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s70
+; VI-NEXT: v_mov_b32_e32 v7, s68
; VI-NEXT: v_add_f16_sdwa v7, v7, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v25
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s68
+; VI-NEXT: v_mov_b32_e32 v8, s34
; VI-NEXT: v_add_f16_sdwa v8, v8, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v25
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s31
+; VI-NEXT: v_mov_b32_e32 v9, vcc_hi
; VI-NEXT: v_add_f16_sdwa v9, v9, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v25
; VI-NEXT: v_or_b32_e32 v9, v10, v9
@@ -38500,17 +39024,17 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v13, v13, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v25
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s80
+; VI-NEXT: v_mov_b32_e32 v14, s70
; VI-NEXT: v_add_f16_sdwa v14, v14, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s71, v25
+; VI-NEXT: v_add_f16_e32 v15, s69, v25
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s69
+; VI-NEXT: v_mov_b32_e32 v15, s35
; VI-NEXT: v_add_f16_sdwa v15, v15, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s35, v25
+; VI-NEXT: v_add_f16_e32 v16, s31, v25
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s34
+; VI-NEXT: v_mov_b32_e32 v16, s30
; VI-NEXT: v_add_f16_sdwa v16, v16, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s30, v25
+; VI-NEXT: v_add_f16_e32 v17, vcc_lo, v25
; VI-NEXT: v_or_b32_e32 v16, v17, v16
; VI-NEXT: v_mov_b32_e32 v17, s91
; VI-NEXT: v_add_f16_sdwa v17, v17, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
@@ -38548,11 +39072,8 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v26, v26, v25 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s6, v25
; VI-NEXT: v_or_b32_e32 v25, v25, v26
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -38585,11 +39106,9 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
-; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s31, v32, 31
-; VI-NEXT: v_readlane_b32 s87, v32, 29
-; VI-NEXT: v_readlane_b32 s86, v32, 28
+; VI-NEXT: .LBB55_6: ; %end
+; VI-NEXT: v_readlane_b32 s30, v32, 28
+; VI-NEXT: v_readlane_b32 s31, v32, 29
; VI-NEXT: v_readlane_b32 s85, v32, 27
; VI-NEXT: v_readlane_b32 s84, v32, 26
; VI-NEXT: v_readlane_b32 s83, v32, 25
@@ -38708,10 +39227,18 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s59, s59, s72
; GFX9-NEXT: s_pack_ll_b32_b16 s60, s60, s63
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s62
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v25, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v25 op_sel_hi:[1,0]
@@ -38739,10 +39266,8 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v23, s59, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v24, s60, v25 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v25, s61, v25 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -38775,7 +39300,7 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -38860,11 +39385,16 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s23, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s24, s47, s60
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s46, s59
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -38892,8 +39422,6 @@ define inreg <13 x double> @bitcast_v52f16_to_v13f64_scalar(<52 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v24, 0x200, s24 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v25, 0x200, s25 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -40212,19 +40740,19 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s84, v1
; SI-NEXT: v_readfirstlane_b32 s86, v0
; SI-NEXT: s_lshr_b32 s54, s29, 16
-; SI-NEXT: s_lshr_b32 s91, s28, 16
+; SI-NEXT: s_lshr_b32 s93, s28, 16
; SI-NEXT: s_lshr_b32 s53, s27, 16
-; SI-NEXT: s_lshr_b32 s89, s26, 16
+; SI-NEXT: s_lshr_b32 s91, s26, 16
; SI-NEXT: s_lshr_b32 s52, s25, 16
-; SI-NEXT: s_lshr_b32 s79, s24, 16
+; SI-NEXT: s_lshr_b32 s89, s24, 16
; SI-NEXT: s_lshr_b32 s51, s23, 16
-; SI-NEXT: s_lshr_b32 s77, s22, 16
+; SI-NEXT: s_lshr_b32 s79, s22, 16
; SI-NEXT: s_lshr_b32 s50, s21, 16
-; SI-NEXT: s_lshr_b32 s75, s20, 16
+; SI-NEXT: s_lshr_b32 s77, s20, 16
; SI-NEXT: s_lshr_b32 s49, s19, 16
-; SI-NEXT: s_lshr_b32 s73, s18, 16
+; SI-NEXT: s_lshr_b32 s75, s18, 16
; SI-NEXT: s_lshr_b32 s48, s17, 16
-; SI-NEXT: s_lshr_b32 s63, s16, 16
+; SI-NEXT: s_lshr_b32 s73, s16, 16
; SI-NEXT: s_lshr_b32 s68, s85, 16
; SI-NEXT: s_lshr_b32 s96, s99, 16
; SI-NEXT: s_lshr_b32 s67, s81, 16
@@ -40234,49 +40762,45 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s65, s98, 16
; SI-NEXT: s_lshr_b32 s31, s69, 16
; SI-NEXT: s_lshr_b32 s64, s87, 16
-; SI-NEXT: s_lshr_b32 s95, s97, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s97, 16
; SI-NEXT: s_lshr_b32 s55, s84, 16
-; SI-NEXT: s_lshr_b32 s93, s86, 16
+; SI-NEXT: s_lshr_b32 s95, s86, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s17, 0xffff
; SI-NEXT: s_lshl_b32 s7, s48, 16
-; SI-NEXT: s_lshl_b32 s62, s63, 16
-; SI-NEXT: s_mov_b32 s9, s63
; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
; SI-NEXT: s_lshl_b32 s7, s49, 16
-; SI-NEXT: s_lshl_b32 s72, s73, 16
-; SI-NEXT: s_mov_b32 s11, s73
+; SI-NEXT: s_lshl_b32 s62, s73, 16
+; SI-NEXT: s_mov_b32 s9, s73
; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
; SI-NEXT: s_lshl_b32 s7, s50, 16
-; SI-NEXT: s_lshl_b32 s74, s75, 16
-; SI-NEXT: s_mov_b32 s90, s75
+; SI-NEXT: s_lshl_b32 s72, s75, 16
+; SI-NEXT: s_mov_b32 s11, s75
; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s51, 16
-; SI-NEXT: s_lshl_b32 s76, s77, 16
-; SI-NEXT: s_lshl_b32 s60, s91, 16
-; SI-NEXT: s_mov_b32 s94, s91
-; SI-NEXT: s_mov_b32 s91, s77
+; SI-NEXT: s_lshl_b32 s74, s77, 16
+; SI-NEXT: s_mov_b32 s90, s77
; SI-NEXT: s_or_b32 s77, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
; SI-NEXT: s_lshl_b32 s7, s52, 16
-; SI-NEXT: s_lshl_b32 s78, s79, 16
-; SI-NEXT: s_mov_b32 s92, s79
+; SI-NEXT: s_lshl_b32 s76, s79, 16
+; SI-NEXT: s_lshl_b32 s88, s91, 16
+; SI-NEXT: s_lshl_b32 s60, s93, 16
+; SI-NEXT: s_mov_b32 s94, s93
+; SI-NEXT: s_mov_b32 s93, s91
+; SI-NEXT: s_mov_b32 s91, s79
; SI-NEXT: s_or_b32 s79, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
; SI-NEXT: s_lshl_b32 s7, s53, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s88, s89, 16
-; SI-NEXT: s_lshl_b32 s58, s93, 16
-; SI-NEXT: s_lshl_b32 s56, s95, 16
-; SI-NEXT: s_mov_b32 s30, s95
-; SI-NEXT: s_mov_b32 s95, s93
-; SI-NEXT: s_mov_b32 s93, s89
+; SI-NEXT: s_lshl_b32 s78, s89, 16
+; SI-NEXT: s_mov_b32 s92, s89
; SI-NEXT: s_or_b32 s89, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s54, 16
@@ -40307,66 +40831,104 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s7, s67, 16
; SI-NEXT: s_mov_b32 s15, s73
; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
+; SI-NEXT: s_mov_b32 s13, s75
+; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
; SI-NEXT: s_or_b32 s6, s4, s88
; SI-NEXT: s_and_b32 s4, s28, 0xffff
; SI-NEXT: s_or_b32 s43, s5, s7
; SI-NEXT: s_and_b32 s5, s85, 0xffff
; SI-NEXT: s_lshl_b32 s7, s68, 16
-; SI-NEXT: s_mov_b32 s41, s63
-; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
-; SI-NEXT: s_mov_b32 s73, s11
-; SI-NEXT: s_mov_b32 s13, s75
-; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
+; SI-NEXT: s_mov_b32 s73, s9
+; SI-NEXT: s_mov_b32 s75, s11
; SI-NEXT: s_mov_b32 s11, s77
; SI-NEXT: s_lshr_b64 s[76:77], s[76:77], 16
-; SI-NEXT: s_or_b32 s4, s4, s60
-; SI-NEXT: s_or_b32 vcc_hi, s5, s7
-; SI-NEXT: s_mov_b32 s63, s9
-; SI-NEXT: s_mov_b32 s75, s90
-; SI-NEXT: s_mov_b32 s77, s91
; SI-NEXT: s_mov_b32 s9, s79
; SI-NEXT: s_lshr_b64 s[78:79], s[78:79], 16
+; SI-NEXT: s_or_b32 s4, s4, s60
+; SI-NEXT: s_lshl_b32 s58, s95, 16
+; SI-NEXT: s_or_b32 s37, s5, s7
+; SI-NEXT: s_mov_b32 s77, s90
+; SI-NEXT: s_mov_b32 s79, s91
; SI-NEXT: s_mov_b32 s7, s89
; SI-NEXT: s_lshr_b64 s[88:89], s[88:89], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[60:61], 16
; SI-NEXT: s_and_b32 s60, s86, 0xffff
-; SI-NEXT: s_mov_b32 s79, s92
-; SI-NEXT: s_mov_b32 s89, s93
+; SI-NEXT: s_lshl_b32 s56, vcc_hi, 16
+; SI-NEXT: s_mov_b32 s41, s63
+; SI-NEXT: s_lshr_b64 s[62:63], s[62:63], 16
+; SI-NEXT: s_mov_b32 s89, s92
+; SI-NEXT: s_mov_b32 s91, s93
; SI-NEXT: s_or_b32 s60, s60, s58
; SI-NEXT: s_lshr_b64 s[92:93], s[58:59], 16
; SI-NEXT: s_and_b32 s58, s97, 0xffff
; SI-NEXT: s_lshl_b32 s46, s31, 16
-; SI-NEXT: s_mov_b32 s91, s94
-; SI-NEXT: s_mov_b32 s93, s95
+; SI-NEXT: s_mov_b32 s93, s94
; SI-NEXT: s_or_b32 s58, s58, s56
+; SI-NEXT: s_mov_b32 s63, s95
; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
; SI-NEXT: s_and_b32 s56, s69, 0xffff
; SI-NEXT: s_lshl_b32 s44, s71, 16
-; SI-NEXT: s_mov_b32 s95, s30
+; SI-NEXT: s_mov_b32 s95, s63
; SI-NEXT: s_or_b32 s56, s56, s46
-; SI-NEXT: s_mov_b32 s34, s31
-; SI-NEXT: s_lshr_b64 s[30:31], s[46:47], 16
+; SI-NEXT: s_mov_b32 s63, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
; SI-NEXT: s_and_b32 s46, s80, 0xffff
; SI-NEXT: s_lshl_b32 s42, s82, 16
-; SI-NEXT: s_mov_b32 s31, s34
+; SI-NEXT: s_mov_b32 vcc_hi, s63
; SI-NEXT: s_or_b32 s46, s46, s44
-; SI-NEXT: s_lshr_b64 s[34:35], s[44:45], 16
+; SI-NEXT: s_mov_b32 s63, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 16
; SI-NEXT: s_and_b32 s44, s83, 0xffff
-; SI-NEXT: s_lshl_b32 vcc_lo, s96, 16
+; SI-NEXT: s_lshl_b32 s36, s96, 16
; SI-NEXT: s_or_b32 s44, s44, s42
-; SI-NEXT: s_lshr_b64 s[36:37], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
; SI-NEXT: s_and_b32 s42, s99, 0xffff
; SI-NEXT: s_mov_b32 s5, s61
; SI-NEXT: s_mov_b32 s61, s59
; SI-NEXT: s_mov_b32 s59, s57
; SI-NEXT: s_mov_b32 s57, s47
; SI-NEXT: s_mov_b32 s47, s45
+; SI-NEXT: s_mov_b32 s31, s63
; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_or_b32 s42, s42, vcc_lo
-; SI-NEXT: s_mov_b32 s43, vcc_hi
-; SI-NEXT: s_lshr_b64 s[38:39], vcc, 16
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_or_b32 s42, s42, s36
+; SI-NEXT: s_mov_b32 s43, s37
+; SI-NEXT: s_lshr_b64 s[36:37], s[36:37], 16
+; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[38:39], -1
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[38:39], s[38:39], exec
+; SI-NEXT: s_cselect_b32 s63, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s63, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s99, s99, 3
; SI-NEXT: s_and_b32 s4, s99, 0xffff
; SI-NEXT: s_lshl_b32 s5, s96, 16
@@ -40409,7 +40971,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s97, s97, 3
; SI-NEXT: s_add_i32 s57, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s95, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s58, s4, 0x30000
@@ -40419,7 +40981,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s86, s86, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s93, 16
+; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
@@ -40429,7 +40991,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s29, 0xffff
@@ -40437,7 +40999,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s89, 16
+; SI-NEXT: s_lshl_b32 s7, s91, 16
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s27, 0xffff
@@ -40445,7 +41007,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s79, 16
+; SI-NEXT: s_lshl_b32 s9, s89, 16
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s25, 0xffff
@@ -40453,7 +41015,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s77, 16
+; SI-NEXT: s_lshl_b32 s11, s79, 16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s23, 0xffff
@@ -40461,7 +41023,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s11, s12, s11
; SI-NEXT: s_and_b32 s12, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s13, s75, 16
+; SI-NEXT: s_lshl_b32 s13, s77, 16
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_and_b32 s13, s21, 0xffff
@@ -40469,7 +41031,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s13, s14, s13
; SI-NEXT: s_and_b32 s14, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s15, s73, 16
+; SI-NEXT: s_lshl_b32 s15, s75, 16
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_and_b32 s15, s19, 0xffff
@@ -40477,7 +41039,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s15, s18, s15
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s63, 16
+; SI-NEXT: s_lshl_b32 s18, s73, 16
; SI-NEXT: s_or_b32 s16, s18, s16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s40, s16, 0x30000
@@ -40506,10 +41068,10 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[60:61], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[58:59], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[56:57], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[42:43], 16
+; SI-NEXT: s_lshr_b64 vcc, s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[42:43], 16
; SI-NEXT: s_lshr_b32 s48, s41, 16
; SI-NEXT: s_lshr_b32 s49, s15, 16
; SI-NEXT: s_lshr_b32 s50, s13, 16
@@ -40523,7 +41085,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_lshr_b32 s66, s47, 16
; SI-NEXT: s_lshr_b32 s67, s45, 16
; SI-NEXT: s_lshr_b32 s68, s43, 16
-; SI-NEXT: .LBB57_3: ; %end
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s16, s40, 0xffff
; SI-NEXT: s_lshl_b32 s17, s62, 16
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -40579,25 +41141,25 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s22, s64, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s23, s30, 16
+; SI-NEXT: s_lshl_b32 s23, vcc_lo, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s57, 0xffff
; SI-NEXT: s_lshl_b32 s24, s65, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s34, 16
+; SI-NEXT: s_lshl_b32 s25, s30, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s47, 0xffff
; SI-NEXT: s_lshl_b32 s26, s66, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s44, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s36, 16
+; SI-NEXT: s_lshl_b32 s27, s34, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s45, 0xffff
; SI-NEXT: s_lshl_b32 s28, s67, 16
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s38, 16
+; SI-NEXT: s_lshl_b32 s29, s36, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s43, 0xffff
; SI-NEXT: s_lshl_b32 s40, s68, 16
@@ -40669,34 +41231,6 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v52i16_to_v52f16_scalar:
; VI: ; %bb.0:
@@ -40741,10 +41275,18 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s78, s77, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -40797,7 +41339,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s9, s9, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s4, s5
@@ -40903,8 +41445,6 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v24, s8
; VI-NEXT: v_mov_b32_e32 v25, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v52i16_to_v52f16_scalar:
; GFX9: ; %bb.0:
@@ -40953,10 +41493,18 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s72, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v12
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s91, s63
; GFX9-NEXT: v_pk_add_u16 v25, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s90, s62
@@ -41035,10 +41583,8 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v23
; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v24
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v25, s91
; GFX9-NEXT: v_mov_b32_e32 v24, s90
; GFX9-NEXT: v_mov_b32_e32 v23, s89
@@ -41091,7 +41637,7 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v41, s8
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -41191,11 +41737,16 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s62, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s78, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s78, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s78, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s74, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s72, s47
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s61, s77, s61
@@ -41275,8 +41826,6 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v25
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s77 :: v_dual_mov_b32 v24, s76
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s74 :: v_dual_mov_b32 v22, s72
@@ -41375,11 +41924,16 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s62, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s78, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s78, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s78, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s74, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s72, s47
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s61, s77, s61
@@ -41459,8 +42013,6 @@ define inreg <52 x half> @bitcast_v52i16_to_v52f16_scalar(<52 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v27, 16, v22
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v21
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s77 :: v_dual_mov_b32 v22, s76
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s74 :: v_dual_mov_b32 v24, s72
@@ -42493,10 +43045,18 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s76, s79, 16
; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s91
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s90
@@ -42735,10 +43295,8 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: v_lshr_b64 v[42:43], v[0:1], 16
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v45, s61
; SI-NEXT: v_mov_b32_e32 v47, s59
; SI-NEXT: v_mov_b32_e32 v29, s56
@@ -42797,7 +43355,7 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v30, s58
; SI-NEXT: v_mov_b32_e32 v55, s47
; SI-NEXT: v_mov_b32_e32 v26, s44
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v42
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -42943,10 +43501,18 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s91, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v12
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v26, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v26
; VI-NEXT: v_add_f16_e32 v43, s43, v26
@@ -43000,10 +43566,8 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v27, s47, v26
; VI-NEXT: v_add_f16_e32 v25, s44, v26
; VI-NEXT: v_add_f16_e32 v26, s45, v26
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v26, s45
; VI-NEXT: v_mov_b32_e32 v25, s44
; VI-NEXT: v_mov_b32_e32 v27, s47
@@ -43056,7 +43620,7 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v43, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v43, 16, v43
; VI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
; VI-NEXT: v_lshlrev_b32_e32 v41, 16, v41
@@ -43163,10 +43727,18 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s72, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v12
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s91, s63
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v25, s4, v0 op_sel_hi:[1,0]
@@ -43246,10 +43818,8 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v23
; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v24
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v25, s91
; GFX9-NEXT: v_mov_b32_e32 v24, s90
; GFX9-NEXT: v_mov_b32_e32 v23, s89
@@ -43302,7 +43872,7 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v41, s8
; GFX9-NEXT: v_mov_b32_e32 v42, s7
; GFX9-NEXT: v_mov_b32_e32 v43, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -43402,11 +43972,16 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s62, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s78, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s78, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s78, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s74, s57
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s47, s72, s47
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s61, s77, s61
@@ -43486,8 +44061,6 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v25
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s77 :: v_dual_mov_b32 v24, s76
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s74 :: v_dual_mov_b32 v22, s72
@@ -43586,11 +44159,16 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s62, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s78, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s78, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s78
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s78, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s78, s78, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s78, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s78, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s74, s57
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s47, s72, s47
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s61, s77, s61
@@ -43670,8 +44248,6 @@ define inreg <52 x i16> @bitcast_v52f16_to_v52i16_scalar(<52 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v27, 16, v22
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v26, 16, v21
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v21, s77 :: v_dual_mov_b32 v22, s76
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s74 :: v_dual_mov_b32 v24, s72
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
index 22b1bc3ed9468..358779c3a73b2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll
@@ -208,10 +208,18 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -240,7 +248,7 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -270,8 +278,6 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v28i32_to_v28f32_scalar:
; VI: ; %bb.0:
@@ -292,10 +298,18 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -324,7 +338,7 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -354,8 +368,6 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v28i32_to_v28f32_scalar:
; GFX9: ; %bb.0:
@@ -376,10 +388,18 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -408,7 +428,7 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -438,8 +458,6 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v28i32_to_v28f32_scalar:
; GFX11: ; %bb.0:
@@ -457,11 +475,16 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -490,7 +513,7 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -507,8 +530,6 @@ define inreg <28 x float> @bitcast_v28i32_to_v28f32_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -744,10 +765,18 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
; SI-NEXT: v_add_f32_e64 v26, s62, 1.0
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -776,10 +805,8 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -812,7 +839,7 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -879,10 +906,18 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
; VI-NEXT: v_add_f32_e64 v26, s62, 1.0
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -911,10 +946,8 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -947,7 +980,7 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -1014,10 +1047,18 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX9-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -1046,10 +1087,8 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -1082,7 +1121,7 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -1150,11 +1189,16 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX11-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -1184,8 +1228,6 @@ define inreg <28 x i32> @bitcast_v28f32_to_v28i32_scalar(<28 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1440,10 +1482,18 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1472,7 +1522,7 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1502,8 +1552,6 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v28i32_to_v14i64_scalar:
; VI: ; %bb.0:
@@ -1524,10 +1572,18 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1556,7 +1612,7 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1586,8 +1642,6 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v28i32_to_v14i64_scalar:
; GFX9: ; %bb.0:
@@ -1608,10 +1662,18 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1640,7 +1702,7 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1670,8 +1732,6 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v28i32_to_v14i64_scalar:
; GFX11: ; %bb.0:
@@ -1689,11 +1749,16 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1722,7 +1787,7 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1739,8 +1804,6 @@ define inreg <14 x i64> @bitcast_v28i32_to_v14i64_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1967,10 +2030,18 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -1999,7 +2070,7 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2029,8 +2100,6 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v14i64_to_v28i32_scalar:
; VI: ; %bb.0:
@@ -2051,10 +2120,18 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -2083,7 +2160,7 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2113,8 +2190,6 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v14i64_to_v28i32_scalar:
; GFX9: ; %bb.0:
@@ -2135,10 +2210,18 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -2167,7 +2250,7 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2197,8 +2280,6 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v14i64_to_v28i32_scalar:
; GFX11: ; %bb.0:
@@ -2216,11 +2297,16 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -2249,7 +2335,7 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2266,8 +2352,6 @@ define inreg <28 x i32> @bitcast_v14i64_to_v28i32_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2487,10 +2571,18 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2519,7 +2611,7 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2549,8 +2641,6 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v28i32_to_v14f64_scalar:
; VI: ; %bb.0:
@@ -2571,10 +2661,18 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2603,7 +2701,7 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2633,8 +2731,6 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v28i32_to_v14f64_scalar:
; GFX9: ; %bb.0:
@@ -2655,10 +2751,18 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2687,7 +2791,7 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2717,8 +2821,6 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v28i32_to_v14f64_scalar:
; GFX11: ; %bb.0:
@@ -2736,11 +2838,16 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -2769,7 +2876,7 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2786,8 +2893,6 @@ define inreg <14 x double> @bitcast_v28i32_to_v14f64_scalar(<28 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2981,10 +3086,18 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -2999,10 +3112,8 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -3035,7 +3146,7 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -3102,10 +3213,18 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -3120,10 +3239,8 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -3156,7 +3273,7 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -3223,10 +3340,18 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -3241,10 +3366,8 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -3277,7 +3400,7 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -3345,11 +3468,16 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -3365,8 +3493,6 @@ define inreg <28 x i32> @bitcast_v14f64_to_v28i32_scalar(<14 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -4273,10 +4399,8 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: v_writelane_b32 v28, s49, 7
; SI-NEXT: v_writelane_b32 v28, s50, 8
; SI-NEXT: v_writelane_b32 v28, s51, 9
-; SI-NEXT: v_writelane_b32 v28, s52, 10
-; SI-NEXT: v_writelane_b32 v28, s53, 11
-; SI-NEXT: v_writelane_b32 v28, s30, 12
-; SI-NEXT: v_writelane_b32 v28, s31, 13
+; SI-NEXT: v_writelane_b32 v28, s30, 10
+; SI-NEXT: v_writelane_b32 v28, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -4293,22 +4417,22 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -4323,8 +4447,44 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -4362,97 +4522,97 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[28:29], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[26:27], 16
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_lshl_b32 s43, s92, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s43
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s53, 16
+; SI-NEXT: s_lshl_b32 s43, s51, 16
; SI-NEXT: s_or_b32 s17, s17, s43
; SI-NEXT: s_lshl_b32 s43, s90, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s43
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s52, 16
+; SI-NEXT: s_lshl_b32 s43, s50, 16
; SI-NEXT: s_or_b32 s19, s19, s43
; SI-NEXT: s_lshl_b32 s43, s88, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s43
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s51, 16
+; SI-NEXT: s_lshl_b32 s43, s49, 16
; SI-NEXT: s_or_b32 s21, s21, s43
; SI-NEXT: s_lshl_b32 s43, s78, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s43
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s50, 16
+; SI-NEXT: s_lshl_b32 s43, s48, 16
; SI-NEXT: s_or_b32 s23, s23, s43
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_lshl_b32 s43, s76, 16
; SI-NEXT: s_or_b32 s24, s24, s43
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s49, 16
+; SI-NEXT: s_lshl_b32 s43, s39, 16
; SI-NEXT: s_or_b32 s25, s25, s43
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s43, s74, 16
; SI-NEXT: s_or_b32 s26, s26, s43
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s48, 16
+; SI-NEXT: s_lshl_b32 s43, s38, 16
; SI-NEXT: s_or_b32 s27, s27, s43
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s43, s72, 16
; SI-NEXT: s_or_b32 s28, s28, s43
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s39, 16
+; SI-NEXT: s_lshl_b32 s43, s37, 16
; SI-NEXT: s_or_b32 s29, s29, s43
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s43, s62, 16
; SI-NEXT: s_or_b32 s40, s40, s43
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s38, 16
+; SI-NEXT: s_lshl_b32 s43, s36, 16
; SI-NEXT: s_or_b32 s41, s41, s43
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s43, s60, 16
; SI-NEXT: s_or_b32 s14, s14, s43
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s37, 16
+; SI-NEXT: s_lshl_b32 s43, s35, 16
; SI-NEXT: s_or_b32 s15, s15, s43
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s43, s58, 16
; SI-NEXT: s_or_b32 s12, s12, s43
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s36, 16
+; SI-NEXT: s_lshl_b32 s43, s34, 16
; SI-NEXT: s_or_b32 s13, s13, s43
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s43, s56, 16
; SI-NEXT: s_or_b32 s10, s10, s43
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s35, 16
+; SI-NEXT: s_lshl_b32 s43, s31, 16
; SI-NEXT: s_or_b32 s11, s11, s43
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s43, s46, 16
; SI-NEXT: s_or_b32 s8, s8, s43
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s34, 16
+; SI-NEXT: s_lshl_b32 s43, s30, 16
; SI-NEXT: s_or_b32 s9, s9, s43
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s43, s44, 16
@@ -4460,13 +4620,13 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_or_b32 s6, s6, s43
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s31, 16
+; SI-NEXT: s_lshl_b32 s43, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s30, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_lo, 16
; SI-NEXT: s_or_b32 s7, s7, s43
; SI-NEXT: s_or_b32 s5, s5, s42
-; SI-NEXT: v_readlane_b32 s30, v28, 12
+; SI-NEXT: v_readlane_b32 s30, v28, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -4495,9 +4655,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v25, s7
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
-; SI-NEXT: v_readlane_b32 s31, v28, 13
-; SI-NEXT: v_readlane_b32 s53, v28, 11
-; SI-NEXT: v_readlane_b32 s52, v28, 10
+; SI-NEXT: v_readlane_b32 s31, v28, 11
; SI-NEXT: v_readlane_b32 s51, v28, 9
; SI-NEXT: v_readlane_b32 s50, v28, 8
; SI-NEXT: v_readlane_b32 s49, v28, 7
@@ -4513,36 +4671,6 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v28i32_to_v56i16_scalar:
; VI: ; %bb.0:
@@ -4550,10 +4678,8 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v28, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v28, s34, 0
-; VI-NEXT: v_writelane_b32 v28, s35, 1
-; VI-NEXT: v_writelane_b32 v28, s30, 2
-; VI-NEXT: v_writelane_b32 v28, s31, 3
+; VI-NEXT: v_writelane_b32 v28, s30, 0
+; VI-NEXT: v_writelane_b32 v28, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -4570,7 +4696,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -4596,12 +4722,48 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -4654,22 +4816,22 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s34, 16
+; VI-NEXT: s_lshl_b32 s16, s30, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s31, 16
+; VI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s30, 16
+; VI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s91, 16
@@ -4743,7 +4905,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s42, s44, 16
; VI-NEXT: s_or_b32 s6, s6, s42
-; VI-NEXT: v_readlane_b32 s30, v28, 2
+; VI-NEXT: v_readlane_b32 s30, v28, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -4772,44 +4934,12 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_readlane_b32 s31, v28, 3
-; VI-NEXT: v_readlane_b32 s35, v28, 1
-; VI-NEXT: v_readlane_b32 s34, v28, 0
+; VI-NEXT: v_readlane_b32 s31, v28, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v28, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v28i32_to_v56i16_scalar:
; GFX9: ; %bb.0:
@@ -4830,7 +4960,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -4860,8 +4990,44 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -4918,7 +5084,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s93
@@ -4976,36 +5142,6 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v28i32_to_v56i16_scalar:
; GFX11: ; %bb.0:
@@ -5023,7 +5159,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s90, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s4, 16
; GFX11-NEXT: s_lshr_b32 s15, s5, 16
@@ -5053,9 +5189,44 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s90, -1
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -5112,7 +5283,7 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s89
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s88
@@ -5157,36 +5328,6 @@ define inreg <56 x i16> @bitcast_v28i32_to_v56i16_scalar(<28 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6485,10 +6626,8 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v28, s83, 25
; SI-NEXT: v_writelane_b32 v28, s84, 26
; SI-NEXT: v_writelane_b32 v28, s85, 27
-; SI-NEXT: v_writelane_b32 v28, s86, 28
-; SI-NEXT: v_writelane_b32 v28, s87, 29
-; SI-NEXT: v_writelane_b32 v28, s30, 30
-; SI-NEXT: v_writelane_b32 v28, s31, 31
+; SI-NEXT: v_writelane_b32 v28, s30, 28
+; SI-NEXT: v_writelane_b32 v28, s31, 29
; SI-NEXT: v_readfirstlane_b32 s7, v13
; SI-NEXT: v_readfirstlane_b32 s9, v12
; SI-NEXT: v_readfirstlane_b32 s11, v11
@@ -6500,23 +6639,23 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s88, v5
; SI-NEXT: v_readfirstlane_b32 s91, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s69, v1
-; SI-NEXT: v_readfirstlane_b32 s80, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; SI-NEXT: v_readfirstlane_b32 s35, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
; SI-NEXT: s_lshr_b32 s78, s29, 16
; SI-NEXT: s_lshr_b32 s89, s28, 16
; SI-NEXT: s_lshr_b32 s92, s27, 16
; SI-NEXT: s_lshr_b32 s95, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s68, s24, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -6528,42 +6667,42 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s79, s88, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s69, 16
-; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s35, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -6577,14 +6716,14 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -6619,56 +6758,65 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -6690,20 +6838,20 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -6761,8 +6909,8 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s63, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v28, 30
+; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v28, 28
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -6791,9 +6939,7 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v25, s61
; SI-NEXT: v_mov_b32_e32 v26, s62
; SI-NEXT: v_mov_b32_e32 v27, s63
-; SI-NEXT: v_readlane_b32 s31, v28, 31
-; SI-NEXT: v_readlane_b32 s87, v28, 29
-; SI-NEXT: v_readlane_b32 s86, v28, 28
+; SI-NEXT: v_readlane_b32 s31, v28, 29
; SI-NEXT: v_readlane_b32 s85, v28, 27
; SI-NEXT: v_readlane_b32 s84, v28, 26
; SI-NEXT: v_readlane_b32 s83, v28, 25
@@ -6827,9 +6973,6 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v56i16_to_v28i32_scalar:
; VI: ; %bb.0:
@@ -6870,54 +7013,54 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v28, s87, 29
; VI-NEXT: v_writelane_b32 v28, s30, 30
; VI-NEXT: v_writelane_b32 v28, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s86, v13
-; VI-NEXT: v_readfirstlane_b32 s6, v12
+; VI-NEXT: v_readfirstlane_b32 s81, v13
+; VI-NEXT: v_readfirstlane_b32 s83, v12
; VI-NEXT: v_readfirstlane_b32 s9, v11
-; VI-NEXT: v_readfirstlane_b32 s8, v10
-; VI-NEXT: v_readfirstlane_b32 s87, v9
-; VI-NEXT: v_readfirstlane_b32 s12, v8
+; VI-NEXT: v_readfirstlane_b32 s84, v10
+; VI-NEXT: v_readfirstlane_b32 s85, v9
+; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s73, v7
-; VI-NEXT: v_readfirstlane_b32 s77, v6
-; VI-NEXT: v_readfirstlane_b32 s88, v5
+; VI-NEXT: v_readfirstlane_b32 s75, v6
+; VI-NEXT: v_readfirstlane_b32 s77, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s81, v1
-; VI-NEXT: v_readfirstlane_b32 s84, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s71, v1
+; VI-NEXT: v_readfirstlane_b32 s82, v0
; VI-NEXT: s_lshr_b32 s79, s29, 16
; VI-NEXT: s_lshr_b32 s90, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s67, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s80, s24, 16
-; VI-NEXT: s_lshr_b32 s82, s23, 16
-; VI-NEXT: s_lshr_b32 s85, s22, 16
-; VI-NEXT: s_lshr_b32 s7, s21, 16
-; VI-NEXT: s_lshr_b32 s11, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s69, s23, 16
+; VI-NEXT: s_lshr_b32 s80, s22, 16
+; VI-NEXT: s_lshr_b32 s86, s21, 16
+; VI-NEXT: s_lshr_b32 s87, s20, 16
; VI-NEXT: s_lshr_b32 s13, s19, 16
; VI-NEXT: s_lshr_b32 s15, s18, 16
-; VI-NEXT: s_lshr_b32 s10, s17, 16
-; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_lshr_b32 s64, s86, 16
-; VI-NEXT: s_lshr_b32 s65, s6, 16
-; VI-NEXT: s_lshr_b32 s66, s9, 16
-; VI-NEXT: s_lshr_b32 s14, s8, 16
-; VI-NEXT: s_lshr_b32 s72, s87, 16
-; VI-NEXT: s_lshr_b32 s74, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: s_lshr_b32 s7, s16, 16
+; VI-NEXT: s_lshr_b32 s64, s81, 16
+; VI-NEXT: s_lshr_b32 s10, s83, 16
+; VI-NEXT: s_lshr_b32 s12, s9, 16
+; VI-NEXT: s_lshr_b32 s14, s84, 16
+; VI-NEXT: s_lshr_b32 s72, s85, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
; VI-NEXT: s_lshr_b32 s76, s73, 16
-; VI-NEXT: s_lshr_b32 s78, s77, 16
-; VI-NEXT: s_lshr_b32 s89, s88, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s68, s69, 16
-; VI-NEXT: s_lshr_b32 s71, s81, 16
-; VI-NEXT: s_lshr_b32 s83, s84, 16
+; VI-NEXT: s_lshr_b32 s78, s75, 16
+; VI-NEXT: s_lshr_b32 s89, s77, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s65, s35, 16
+; VI-NEXT: s_lshr_b32 s67, s71, 16
+; VI-NEXT: s_lshr_b32 s70, s82, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
; VI-NEXT: s_lshl_b32 s5, s15, 16
@@ -6925,32 +7068,32 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s19
; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s10, 16
+; VI-NEXT: s_lshl_b32 s41, s88, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -6958,58 +7101,146 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s84
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s82
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s81
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s71
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s88
+; VI-NEXT: s_and_b32 s4, 0xffff, s77
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s77
+; VI-NEXT: s_and_b32 s4, 0xffff, s75
; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s73
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_and_b32 s4, 0xffff, s85
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_and_b32 s4, 0xffff, s84
; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: v_writelane_b32 v29, s26, 0
+; VI-NEXT: s_mov_b32 s26, s27
+; VI-NEXT: s_mov_b32 s27, s77
+; VI-NEXT: v_writelane_b32 v29, s91, 1
+; VI-NEXT: s_mov_b32 s91, s88
+; VI-NEXT: s_mov_b32 s88, s28
+; VI-NEXT: s_mov_b32 s28, s75
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s81
+; VI-NEXT: s_mov_b32 s81, s68
+; VI-NEXT: s_mov_b32 s68, s34
+; VI-NEXT: s_mov_b32 s34, s31
+; VI-NEXT: s_mov_b32 s31, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s90
+; VI-NEXT: s_mov_b32 s90, s89
+; VI-NEXT: s_mov_b32 s89, s79
+; VI-NEXT: s_mov_b32 s79, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s72
+; VI-NEXT: s_mov_b32 s72, s14
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s10, s64
+; VI-NEXT: s_mov_b32 s9, s83
+; VI-NEXT: s_mov_b32 s83, s69
+; VI-NEXT: s_mov_b32 s69, s65
+; VI-NEXT: s_mov_b32 s6, s13
+; VI-NEXT: s_mov_b32 s13, s84
+; VI-NEXT: s_mov_b32 s84, s70
+; VI-NEXT: s_mov_b32 s70, s66
+; VI-NEXT: s_mov_b32 s75, s73
+; VI-NEXT: s_mov_b32 s73, s8
+; VI-NEXT: s_mov_b32 s8, s15
+; VI-NEXT: s_mov_b32 s15, s85
+; VI-NEXT: s_mov_b32 s85, s80
+; VI-NEXT: s_mov_b32 s80, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s80
+; VI-NEXT: s_mov_b32 s80, s85
+; VI-NEXT: s_mov_b32 s85, s15
+; VI-NEXT: s_mov_b32 s15, s8
+; VI-NEXT: s_mov_b32 s8, s73
+; VI-NEXT: s_mov_b32 s73, s75
+; VI-NEXT: s_mov_b32 s66, s70
+; VI-NEXT: s_mov_b32 s70, s84
+; VI-NEXT: s_mov_b32 s84, s13
+; VI-NEXT: s_mov_b32 s13, s6
+; VI-NEXT: s_mov_b32 s65, s69
+; VI-NEXT: s_mov_b32 s69, s83
+; VI-NEXT: s_mov_b32 s83, s9
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s64, s10
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s14, s72
+; VI-NEXT: s_mov_b32 s72, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s79
+; VI-NEXT: s_mov_b32 s79, s89
+; VI-NEXT: s_mov_b32 s89, s90
+; VI-NEXT: s_mov_b32 s90, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s31
+; VI-NEXT: s_mov_b32 s31, s34
+; VI-NEXT: s_mov_b32 s34, s68
+; VI-NEXT: s_mov_b32 s68, s81
+; VI-NEXT: s_mov_b32 s81, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s75, s28
+; VI-NEXT: s_mov_b32 s28, s88
+; VI-NEXT: s_mov_b32 s88, s91
+; VI-NEXT: v_readlane_b32 s91, v29, 1
+; VI-NEXT: s_mov_b32 s77, s27
+; VI-NEXT: s_mov_b32 s27, s26
+; VI-NEXT: v_readlane_b32 s26, v29, 0
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s10, s10, 16
-; VI-NEXT: s_or_b32 s5, s10, s5
+; VI-NEXT: s_lshl_b32 s6, s88, 16
+; VI-NEXT: s_or_b32 s5, s6, s5
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_add_i32 s37, s5, 0x30000
@@ -7024,42 +7255,42 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -7071,37 +7302,37 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s84, s84, 3
+; VI-NEXT: s_add_i32 s82, s82, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s84, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, s82, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s81, s81, 3
+; VI-NEXT: s_add_i32 s71, s71, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s81, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, s71, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s69, s69, 3
+; VI-NEXT: s_add_i32 s35, s35, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s69, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, s35, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s91, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s88, s88, 3
+; VI-NEXT: s_add_i32 s88, s77, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s88, 0xffff
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s77, s77, 3
+; VI-NEXT: s_add_i32 s77, s75, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
; VI-NEXT: s_lshl_b32 s5, s78, 16
@@ -7111,17 +7342,17 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s75, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s12, 3
+; VI-NEXT: s_add_i32 s73, s8, 3
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s87, 3
+; VI-NEXT: s_add_i32 s15, s85, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s8, 3
+; VI-NEXT: s_add_i32 s13, s84, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
; VI-NEXT: s_lshl_b32 s5, s14, 16
@@ -7129,20 +7360,20 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s11, s9, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s6, 3
+; VI-NEXT: s_add_i32 s9, s83, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s86, 3
+; VI-NEXT: s_add_i32 s7, s81, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s63, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v28, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -7209,96 +7440,6 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
-; VI-NEXT: s_mov_b32 vcc_lo, s64
-; VI-NEXT: v_writelane_b32 v29, s75, 0
-; VI-NEXT: s_mov_b32 vcc_hi, s65
-; VI-NEXT: s_mov_b32 s75, s73
-; VI-NEXT: s_mov_b32 s73, s12
-; VI-NEXT: v_writelane_b32 v29, s25, 1
-; VI-NEXT: s_mov_b32 s25, s91
-; VI-NEXT: s_mov_b32 s91, s26
-; VI-NEXT: s_mov_b32 s26, s27
-; VI-NEXT: s_mov_b32 s27, s88
-; VI-NEXT: s_mov_b32 s88, s28
-; VI-NEXT: s_mov_b32 s28, s77
-; VI-NEXT: s_mov_b32 s77, s29
-; VI-NEXT: s_mov_b32 s29, s15
-; VI-NEXT: s_mov_b32 s15, s87
-; VI-NEXT: s_mov_b32 s87, s85
-; VI-NEXT: s_mov_b32 s85, s82
-; VI-NEXT: s_mov_b32 s82, s71
-; VI-NEXT: s_mov_b32 s71, s68
-; VI-NEXT: s_mov_b32 s68, s35
-; VI-NEXT: s_mov_b32 s35, s31
-; VI-NEXT: s_mov_b32 s31, s30
-; VI-NEXT: s_mov_b32 s30, s90
-; VI-NEXT: s_mov_b32 s90, s89
-; VI-NEXT: s_mov_b32 s89, s79
-; VI-NEXT: s_mov_b32 s79, s78
-; VI-NEXT: s_mov_b32 s78, s76
-; VI-NEXT: s_mov_b32 s76, s74
-; VI-NEXT: s_mov_b32 s74, s72
-; VI-NEXT: s_mov_b32 s72, s14
-; VI-NEXT: s_mov_b32 s14, s66
-; VI-NEXT: s_mov_b32 s12, s10
-; VI-NEXT: s_mov_b32 s10, s13
-; VI-NEXT: s_mov_b32 s13, s8
-; VI-NEXT: s_mov_b32 s8, s11
-; VI-NEXT: s_mov_b32 s11, s9
-; VI-NEXT: s_mov_b32 s9, s6
-; VI-NEXT: s_mov_b32 s6, s7
-; VI-NEXT: s_mov_b32 s7, s86
-; VI-NEXT: s_mov_b32 s86, s83
-; VI-NEXT: s_mov_b32 s83, s80
-; VI-NEXT: s_mov_b32 s80, s70
-; VI-NEXT: s_mov_b32 s70, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, s70
-; VI-NEXT: s_mov_b32 s70, s80
-; VI-NEXT: s_mov_b32 s80, s83
-; VI-NEXT: s_mov_b32 s83, s86
-; VI-NEXT: s_mov_b32 s86, s7
-; VI-NEXT: s_mov_b32 s7, s6
-; VI-NEXT: s_mov_b32 s6, s9
-; VI-NEXT: s_mov_b32 s9, s11
-; VI-NEXT: s_mov_b32 s11, s8
-; VI-NEXT: s_mov_b32 s8, s13
-; VI-NEXT: s_mov_b32 s13, s10
-; VI-NEXT: s_mov_b32 s10, s12
-; VI-NEXT: s_mov_b32 s66, s14
-; VI-NEXT: s_mov_b32 s14, s72
-; VI-NEXT: s_mov_b32 s72, s74
-; VI-NEXT: s_mov_b32 s74, s76
-; VI-NEXT: s_mov_b32 s76, s78
-; VI-NEXT: s_mov_b32 s78, s79
-; VI-NEXT: s_mov_b32 s79, s89
-; VI-NEXT: s_mov_b32 s89, s90
-; VI-NEXT: s_mov_b32 s90, s30
-; VI-NEXT: s_mov_b32 s30, s31
-; VI-NEXT: s_mov_b32 s31, s35
-; VI-NEXT: s_mov_b32 s35, s68
-; VI-NEXT: s_mov_b32 s68, s71
-; VI-NEXT: s_mov_b32 s71, s82
-; VI-NEXT: s_mov_b32 s82, s85
-; VI-NEXT: s_mov_b32 s85, s87
-; VI-NEXT: s_mov_b32 s87, s15
-; VI-NEXT: s_mov_b32 s15, s29
-; VI-NEXT: s_mov_b32 s29, s77
-; VI-NEXT: s_mov_b32 s77, s28
-; VI-NEXT: s_mov_b32 s28, s88
-; VI-NEXT: s_mov_b32 s88, s27
-; VI-NEXT: s_mov_b32 s27, s26
-; VI-NEXT: s_mov_b32 s26, s91
-; VI-NEXT: s_mov_b32 s91, s25
-; VI-NEXT: v_readlane_b32 s25, v29, 1
-; VI-NEXT: s_mov_b32 s12, s73
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: v_readlane_b32 s75, v29, 0
-; VI-NEXT: s_mov_b32 s65, vcc_hi
-; VI-NEXT: s_mov_b32 s64, vcc_lo
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v56i16_to_v28i32_scalar:
; GFX9: ; %bb.0:
@@ -7390,10 +7531,18 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -7422,10 +7571,8 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v26, s62, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -7458,7 +7605,7 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -7549,11 +7696,16 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -7583,8 +7735,6 @@ define inreg <28 x i32> @bitcast_v56i16_to_v28i32_scalar(<56 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v26, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v27, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -8474,10 +8624,8 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: v_writelane_b32 v28, s49, 7
; SI-NEXT: v_writelane_b32 v28, s50, 8
; SI-NEXT: v_writelane_b32 v28, s51, 9
-; SI-NEXT: v_writelane_b32 v28, s52, 10
-; SI-NEXT: v_writelane_b32 v28, s53, 11
-; SI-NEXT: v_writelane_b32 v28, s30, 12
-; SI-NEXT: v_writelane_b32 v28, s31, 13
+; SI-NEXT: v_writelane_b32 v28, s30, 10
+; SI-NEXT: v_writelane_b32 v28, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -8494,22 +8642,22 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -8524,8 +8672,44 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -8563,97 +8747,97 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[72:73], s[28:29], 16
; SI-NEXT: s_lshr_b64 s[74:75], s[26:27], 16
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[76:77], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_lshl_b32 s43, s92, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s43
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s53, 16
+; SI-NEXT: s_lshl_b32 s43, s51, 16
; SI-NEXT: s_or_b32 s17, s17, s43
; SI-NEXT: s_lshl_b32 s43, s90, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s43
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s52, 16
+; SI-NEXT: s_lshl_b32 s43, s50, 16
; SI-NEXT: s_or_b32 s19, s19, s43
; SI-NEXT: s_lshl_b32 s43, s88, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s43
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s51, 16
+; SI-NEXT: s_lshl_b32 s43, s49, 16
; SI-NEXT: s_or_b32 s21, s21, s43
; SI-NEXT: s_lshl_b32 s43, s78, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s43
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s50, 16
+; SI-NEXT: s_lshl_b32 s43, s48, 16
; SI-NEXT: s_or_b32 s23, s23, s43
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_lshl_b32 s43, s76, 16
; SI-NEXT: s_or_b32 s24, s24, s43
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s49, 16
+; SI-NEXT: s_lshl_b32 s43, s39, 16
; SI-NEXT: s_or_b32 s25, s25, s43
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s43, s74, 16
; SI-NEXT: s_or_b32 s26, s26, s43
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s48, 16
+; SI-NEXT: s_lshl_b32 s43, s38, 16
; SI-NEXT: s_or_b32 s27, s27, s43
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s43, s72, 16
; SI-NEXT: s_or_b32 s28, s28, s43
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s39, 16
+; SI-NEXT: s_lshl_b32 s43, s37, 16
; SI-NEXT: s_or_b32 s29, s29, s43
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s43, s62, 16
; SI-NEXT: s_or_b32 s40, s40, s43
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s38, 16
+; SI-NEXT: s_lshl_b32 s43, s36, 16
; SI-NEXT: s_or_b32 s41, s41, s43
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s43, s60, 16
; SI-NEXT: s_or_b32 s14, s14, s43
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s37, 16
+; SI-NEXT: s_lshl_b32 s43, s35, 16
; SI-NEXT: s_or_b32 s15, s15, s43
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s43, s58, 16
; SI-NEXT: s_or_b32 s12, s12, s43
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s36, 16
+; SI-NEXT: s_lshl_b32 s43, s34, 16
; SI-NEXT: s_or_b32 s13, s13, s43
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s43, s56, 16
; SI-NEXT: s_or_b32 s10, s10, s43
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s35, 16
+; SI-NEXT: s_lshl_b32 s43, s31, 16
; SI-NEXT: s_or_b32 s11, s11, s43
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s43, s46, 16
; SI-NEXT: s_or_b32 s8, s8, s43
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s34, 16
+; SI-NEXT: s_lshl_b32 s43, s30, 16
; SI-NEXT: s_or_b32 s9, s9, s43
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s43, s44, 16
@@ -8661,13 +8845,13 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_or_b32 s6, s6, s43
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s31, 16
+; SI-NEXT: s_lshl_b32 s43, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s30, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_lo, 16
; SI-NEXT: s_or_b32 s7, s7, s43
; SI-NEXT: s_or_b32 s5, s5, s42
-; SI-NEXT: v_readlane_b32 s30, v28, 12
+; SI-NEXT: v_readlane_b32 s30, v28, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -8696,9 +8880,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v25, s7
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
-; SI-NEXT: v_readlane_b32 s31, v28, 13
-; SI-NEXT: v_readlane_b32 s53, v28, 11
-; SI-NEXT: v_readlane_b32 s52, v28, 10
+; SI-NEXT: v_readlane_b32 s31, v28, 11
; SI-NEXT: v_readlane_b32 s51, v28, 9
; SI-NEXT: v_readlane_b32 s50, v28, 8
; SI-NEXT: v_readlane_b32 s49, v28, 7
@@ -8714,36 +8896,6 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v28i32_to_v56f16_scalar:
; VI: ; %bb.0:
@@ -8751,10 +8903,8 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v28, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v28, s34, 0
-; VI-NEXT: v_writelane_b32 v28, s35, 1
-; VI-NEXT: v_writelane_b32 v28, s30, 2
-; VI-NEXT: v_writelane_b32 v28, s31, 3
+; VI-NEXT: v_writelane_b32 v28, s30, 0
+; VI-NEXT: v_writelane_b32 v28, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -8771,7 +8921,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -8797,12 +8947,48 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -8855,22 +9041,22 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s34, 16
+; VI-NEXT: s_lshl_b32 s16, s30, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s31, 16
+; VI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s30, 16
+; VI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s91, 16
@@ -8944,7 +9130,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s42, s44, 16
; VI-NEXT: s_or_b32 s6, s6, s42
-; VI-NEXT: v_readlane_b32 s30, v28, 2
+; VI-NEXT: v_readlane_b32 s30, v28, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -8973,44 +9159,12 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_readlane_b32 s31, v28, 3
-; VI-NEXT: v_readlane_b32 s35, v28, 1
-; VI-NEXT: v_readlane_b32 s34, v28, 0
+; VI-NEXT: v_readlane_b32 s31, v28, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v28, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v28i32_to_v56f16_scalar:
; GFX9: ; %bb.0:
@@ -9031,7 +9185,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -9061,8 +9215,44 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -9119,7 +9309,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s93
@@ -9177,36 +9367,6 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v28i32_to_v56f16_scalar:
; GFX11: ; %bb.0:
@@ -9224,7 +9384,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s90, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s4, 16
; GFX11-NEXT: s_lshr_b32 s15, s5, 16
@@ -9254,9 +9414,44 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s90, -1
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -9313,7 +9508,7 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s89
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s88
@@ -9358,36 +9553,6 @@ define inreg <56 x half> @bitcast_v28i32_to_v56f16_scalar(<28 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10823,10 +10988,8 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s83, 25
; SI-NEXT: v_writelane_b32 v32, s84, 26
; SI-NEXT: v_writelane_b32 v32, s85, 27
-; SI-NEXT: v_writelane_b32 v32, s86, 28
-; SI-NEXT: v_writelane_b32 v32, s87, 29
-; SI-NEXT: v_writelane_b32 v32, s30, 30
-; SI-NEXT: v_writelane_b32 v32, s31, 31
+; SI-NEXT: v_writelane_b32 v32, s30, 28
+; SI-NEXT: v_writelane_b32 v32, s31, 29
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: v_readfirstlane_b32 s8, v12
; SI-NEXT: v_readfirstlane_b32 s10, v11
@@ -10840,21 +11003,21 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s92, v2
; SI-NEXT: v_readfirstlane_b32 s95, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
+; SI-NEXT: v_readfirstlane_b32 s30, v0
; SI-NEXT: s_lshr_b32 s94, s29, 16
-; SI-NEXT: s_lshr_b32 s30, s28, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s70, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s80, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s28, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s68, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s70, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -10867,59 +11030,59 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s89, s88, 16
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
-; SI-NEXT: s_lshr_b32 s31, s95, 16
-; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s95, 16
+; SI-NEXT: s_lshr_b32 s34, s30, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s50, s4, s5
; SI-NEXT: s_and_b32 s4, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s92, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -10957,11 +11120,20 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s87
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s85
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s84
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -10971,14 +11143,14 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s82
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -10988,7 +11160,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s81
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -10997,11 +11169,11 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s80
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_cvt_f32_f16_e32 v6, s21
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s71
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
@@ -11012,7 +11184,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s70
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
@@ -11021,11 +11193,11 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s69
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: v_cvt_f32_f16_e32 v9, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s68
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
@@ -11036,7 +11208,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s35
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
@@ -11045,11 +11217,11 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: v_cvt_f32_f16_e32 v12, s27
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_lo
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
@@ -11069,11 +11241,11 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_or_b32_e32 v12, v14, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s34
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s34
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v16, vcc_hi
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -11183,11 +11355,8 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v26, v27, v26
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
; SI-NEXT: v_or_b32_e32 v27, v29, v27
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11220,11 +11389,9 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 30
-; SI-NEXT: v_readlane_b32 s31, v32, 31
-; SI-NEXT: v_readlane_b32 s87, v32, 29
-; SI-NEXT: v_readlane_b32 s86, v32, 28
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 28
+; SI-NEXT: v_readlane_b32 s31, v32, 29
; SI-NEXT: v_readlane_b32 s85, v32, 27
; SI-NEXT: v_readlane_b32 s84, v32, 26
; SI-NEXT: v_readlane_b32 s83, v32, 25
@@ -11298,12 +11465,9 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
-; VI-NEXT: s_lshr_b32 s15, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v11
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: v_readfirstlane_b32 s6, v13
-; VI-NEXT: s_lshr_b32 s61, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v10
; VI-NEXT: v_readfirstlane_b32 s14, v9
; VI-NEXT: v_readfirstlane_b32 s72, v8
@@ -11311,178 +11475,198 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s76, v6
; VI-NEXT: v_readfirstlane_b32 s79, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s80, v1
-; VI-NEXT: v_readfirstlane_b32 s83, v0
-; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s70, v1
+; VI-NEXT: v_readfirstlane_b32 s81, v0
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s75, s28, 16
+; VI-NEXT: s_lshr_b32 s61, s28, 16
; VI-NEXT: s_lshr_b32 s90, s27, 16
-; VI-NEXT: s_lshr_b32 s31, s26, 16
-; VI-NEXT: s_lshr_b32 s68, s25, 16
-; VI-NEXT: s_lshr_b32 s70, s24, 16
-; VI-NEXT: s_lshr_b32 s81, s23, 16
-; VI-NEXT: s_lshr_b32 s84, s22, 16
-; VI-NEXT: s_lshr_b32 s86, s21, 16
-; VI-NEXT: s_lshr_b32 s87, s20, 16
-; VI-NEXT: s_lshr_b32 s7, s19, 16
-; VI-NEXT: s_lshr_b32 s9, s18, 16
-; VI-NEXT: s_lshr_b32 s11, s17, 16
-; VI-NEXT: s_lshr_b32 s13, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; VI-NEXT: s_lshr_b32 s34, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s71, s23, 16
+; VI-NEXT: s_lshr_b32 s82, s22, 16
+; VI-NEXT: s_lshr_b32 s84, s21, 16
+; VI-NEXT: s_lshr_b32 s85, s20, 16
+; VI-NEXT: s_lshr_b32 s86, s19, 16
+; VI-NEXT: s_lshr_b32 s87, s18, 16
+; VI-NEXT: s_lshr_b32 s7, s17, 16
+; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s63, s6, 16
-; VI-NEXT: s_lshr_b32 s60, s12, 16
+; VI-NEXT: s_lshr_b32 s11, s8, 16
+; VI-NEXT: s_lshr_b32 s13, s10, 16
+; VI-NEXT: s_lshr_b32 s15, s12, 16
; VI-NEXT: s_lshr_b32 s59, s14, 16
; VI-NEXT: s_lshr_b32 s58, s72, 16
; VI-NEXT: s_lshr_b32 s57, s74, 16
-; VI-NEXT: s_lshr_b32 s62, s76, 16
+; VI-NEXT: s_lshr_b32 s60, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s71, s69, 16
-; VI-NEXT: s_lshr_b32 s82, s80, 16
-; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s69, s35, 16
+; VI-NEXT: s_lshr_b32 s80, s70, 16
+; VI-NEXT: s_lshr_b32 s83, s81, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
-; VI-NEXT: v_writelane_b32 v33, s61, 1
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s60, 2
-; VI-NEXT: v_writelane_b32 v33, s59, 3
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_writelane_b32 v33, s59, 1
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s11, 16
+; VI-NEXT: s_lshl_b32 s41, s7, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s61, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s83
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s80
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s70
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
-; VI-NEXT: s_lshl_b32 s5, s62, 16
-; VI-NEXT: s_mov_b32 s77, s56
+; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_mov_b32 s73, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s75, s57
+; VI-NEXT: s_mov_b32 s78, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s72
; VI-NEXT: s_lshl_b32 s5, s58, 16
-; VI-NEXT: s_mov_b32 s78, s58
+; VI-NEXT: s_mov_b32 s77, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s14
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s59, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s12
-; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_mov_b32 s88, s60
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s10
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_mov_b32 s75, s61
; VI-NEXT: s_or_b32 s61, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, s15, 16
-; VI-NEXT: s_mov_b32 s88, s62
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s62, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s63, 16
-; VI-NEXT: s_mov_b32 s15, s63
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b32 s75, s61
+; VI-NEXT: s_mov_b32 s88, s60
+; VI-NEXT: s_mov_b32 s73, s56
+; VI-NEXT: s_mov_b32 s78, s57
+; VI-NEXT: s_mov_b32 s77, s58
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v27, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s13
-; VI-NEXT: v_mov_b32_e32 v2, s11
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_add_f16_sdwa v0, v0, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v27
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v27
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s9
+; VI-NEXT: v_mov_b32_e32 v2, s87
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v27
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_add_f16_sdwa v3, v3, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v27
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s87
+; VI-NEXT: v_mov_b32_e32 v4, s85
; VI-NEXT: v_add_f16_sdwa v4, v4, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v27
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s86
+; VI-NEXT: v_mov_b32_e32 v5, s84
; VI-NEXT: v_add_f16_sdwa v5, v5, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v27
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s84
+; VI-NEXT: v_mov_b32_e32 v6, s82
; VI-NEXT: v_add_f16_sdwa v6, v6, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v27
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s81
+; VI-NEXT: v_mov_b32_e32 v7, s71
; VI-NEXT: v_add_f16_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v27
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s70
+; VI-NEXT: v_mov_b32_e32 v8, s68
; VI-NEXT: v_add_f16_sdwa v8, v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v27
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s68
+; VI-NEXT: v_mov_b32_e32 v9, s34
; VI-NEXT: v_add_f16_sdwa v9, v9, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v27
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s31
+; VI-NEXT: v_mov_b32_e32 v10, vcc_hi
; VI-NEXT: v_add_f16_sdwa v10, v10, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v27
; VI-NEXT: v_or_b32_e32 v10, v11, v10
@@ -11490,31 +11674,31 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v11, v11, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v27
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s73
+; VI-NEXT: v_mov_b32_e32 v12, s75
; VI-NEXT: v_add_f16_sdwa v12, v12, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v27
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s77
+; VI-NEXT: v_mov_b32_e32 v13, s73
; VI-NEXT: v_add_f16_sdwa v13, v13, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v27
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s85
+; VI-NEXT: v_mov_b32_e32 v14, s83
; VI-NEXT: v_add_f16_sdwa v14, v14, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s83, v27
+; VI-NEXT: v_add_f16_e32 v15, s81, v27
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s82
+; VI-NEXT: v_mov_b32_e32 v15, s80
; VI-NEXT: v_add_f16_sdwa v15, v15, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s80, v27
+; VI-NEXT: v_add_f16_e32 v16, s70, v27
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s71
+; VI-NEXT: v_mov_b32_e32 v16, s69
; VI-NEXT: v_add_f16_sdwa v16, v16, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s69, v27
+; VI-NEXT: v_add_f16_e32 v17, s35, v27
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s35
+; VI-NEXT: v_mov_b32_e32 v17, s31
; VI-NEXT: v_add_f16_sdwa v17, v17, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s34, v27
+; VI-NEXT: v_add_f16_e32 v18, s30, v27
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s30
+; VI-NEXT: v_mov_b32_e32 v18, vcc_lo
; VI-NEXT: v_add_f16_sdwa v18, v18, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s91, v27
; VI-NEXT: v_or_b32_e32 v18, v19, v18
@@ -11526,49 +11710,38 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v20, v20, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s76, v27
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s75
+; VI-NEXT: v_mov_b32_e32 v21, s78
; VI-NEXT: v_add_f16_sdwa v21, v21, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s74, v27
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s78
+; VI-NEXT: v_mov_b32_e32 v22, s77
; VI-NEXT: v_add_f16_sdwa v22, v22, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s72, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s14, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s12, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v24, v25, v24
-; VI-NEXT: v_mov_b32_e32 v25, s4
+; VI-NEXT: v_mov_b32_e32 v25, s15
; VI-NEXT: v_add_f16_sdwa v25, v25, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s10, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v25, v26, v25
-; VI-NEXT: v_mov_b32_e32 v26, s4
+; VI-NEXT: v_mov_b32_e32 v26, s13
; VI-NEXT: v_add_f16_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s8, v27
; VI-NEXT: v_or_b32_e32 v26, v28, v26
-; VI-NEXT: v_mov_b32_e32 v28, s15
+; VI-NEXT: v_mov_b32_e32 v28, s11
; VI-NEXT: v_add_f16_sdwa v28, v28, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s6, v27
; VI-NEXT: v_or_b32_e32 v27, v27, v28
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s88, s62
-; VI-NEXT: s_mov_b32 s77, s56
-; VI-NEXT: s_mov_b32 s75, s57
-; VI-NEXT: s_mov_b32 s78, s58
-; VI-NEXT: s_mov_b32 s15, s63
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -11601,7 +11774,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
+; VI-NEXT: .LBB19_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -11731,10 +11904,18 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v27, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v27 op_sel_hi:[1,0]
@@ -11764,10 +11945,8 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v25, s61, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v26, s62, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v27, s63, v27 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -11800,7 +11979,7 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -11891,11 +12070,16 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -11925,8 +12109,6 @@ define inreg <28 x i32> @bitcast_v56f16_to_v28i32_scalar(<56 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v26, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v27, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12180,10 +12362,18 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
; SI-NEXT: v_add_f32_e64 v26, s62, 1.0
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -12212,10 +12402,8 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -12248,7 +12436,7 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12315,10 +12503,18 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
; VI-NEXT: v_add_f32_e64 v26, s62, 1.0
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -12347,10 +12543,8 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -12383,7 +12577,7 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -12450,10 +12644,18 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX9-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -12482,10 +12684,8 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -12518,7 +12718,7 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -12586,11 +12786,16 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX11-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -12620,8 +12825,6 @@ define inreg <14 x i64> @bitcast_v28f32_to_v14i64_scalar(<28 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -12883,10 +13086,18 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -12915,7 +13126,7 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -12945,8 +13156,6 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v14i64_to_v28f32_scalar:
; VI: ; %bb.0:
@@ -12967,10 +13176,18 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -12999,7 +13216,7 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13029,8 +13246,6 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v14i64_to_v28f32_scalar:
; GFX9: ; %bb.0:
@@ -13051,10 +13266,18 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -13083,7 +13306,7 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13113,8 +13336,6 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v14i64_to_v28f32_scalar:
; GFX11: ; %bb.0:
@@ -13132,11 +13353,16 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -13165,7 +13391,7 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13182,8 +13408,6 @@ define inreg <28 x float> @bitcast_v14i64_to_v28f32_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -13419,10 +13643,18 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
; SI-NEXT: v_add_f32_e64 v26, s62, 1.0
; SI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -13451,10 +13683,8 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -13487,7 +13717,7 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -13554,10 +13784,18 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
; VI-NEXT: v_add_f32_e64 v26, s62, 1.0
; VI-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -13586,10 +13824,8 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -13622,7 +13858,7 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -13689,10 +13925,18 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX9-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX9-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -13721,10 +13965,8 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -13757,7 +13999,7 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -13825,11 +14067,16 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
; GFX11-NEXT: v_add_f32_e64 v26, s62, 1.0
; GFX11-NEXT: v_add_f32_e64 v25, s61, 1.0
@@ -13859,8 +14106,6 @@ define inreg <14 x double> @bitcast_v28f32_to_v14f64_scalar(<28 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -14089,10 +14334,18 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -14107,10 +14360,8 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -14143,7 +14394,7 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -14210,10 +14461,18 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -14228,10 +14487,8 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -14264,7 +14521,7 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -14331,10 +14588,18 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -14349,10 +14614,8 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -14385,7 +14648,7 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -14453,11 +14716,16 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
@@ -14473,8 +14741,6 @@ define inreg <28 x float> @bitcast_v14f64_to_v28f32_scalar(<14 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -15362,10 +15628,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v58, s49, 7
; SI-NEXT: v_writelane_b32 v58, s50, 8
; SI-NEXT: v_writelane_b32 v58, s51, 9
-; SI-NEXT: v_writelane_b32 v58, s52, 10
-; SI-NEXT: v_writelane_b32 v58, s53, 11
-; SI-NEXT: v_writelane_b32 v58, s30, 12
-; SI-NEXT: v_writelane_b32 v58, s31, 13
+; SI-NEXT: v_writelane_b32 v58, s30, 10
+; SI-NEXT: v_writelane_b32 v58, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -15382,22 +15646,22 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s53, s5, 16
-; SI-NEXT: s_lshr_b32 s52, s7, 16
-; SI-NEXT: s_lshr_b32 s51, s9, 16
-; SI-NEXT: s_lshr_b32 s50, s11, 16
-; SI-NEXT: s_lshr_b32 s49, s13, 16
-; SI-NEXT: s_lshr_b32 s48, s15, 16
-; SI-NEXT: s_lshr_b32 s39, s41, 16
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s51, s5, 16
+; SI-NEXT: s_lshr_b32 s50, s7, 16
+; SI-NEXT: s_lshr_b32 s49, s9, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s39, s13, 16
+; SI-NEXT: s_lshr_b32 s38, s15, 16
+; SI-NEXT: s_lshr_b32 s37, s41, 16
+; SI-NEXT: s_lshr_b32 s36, s29, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s23, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -15412,8 +15676,44 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v27, s5, 1.0
; SI-NEXT: v_add_f32_e64 v26, s4, 1.0
; SI-NEXT: v_add_f32_e64 v25, s7, 1.0
@@ -15477,38 +15777,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -15538,20 +15808,20 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v57, s30
-; SI-NEXT: v_mov_b32_e32 v56, s31
-; SI-NEXT: v_mov_b32_e32 v47, s34
-; SI-NEXT: v_mov_b32_e32 v46, s35
-; SI-NEXT: v_mov_b32_e32 v45, s36
-; SI-NEXT: v_mov_b32_e32 v44, s37
-; SI-NEXT: v_mov_b32_e32 v43, s38
-; SI-NEXT: v_mov_b32_e32 v42, s39
-; SI-NEXT: v_mov_b32_e32 v41, s48
-; SI-NEXT: v_mov_b32_e32 v40, s49
-; SI-NEXT: v_mov_b32_e32 v55, s50
-; SI-NEXT: v_mov_b32_e32 v54, s51
-; SI-NEXT: v_mov_b32_e32 v53, s52
-; SI-NEXT: v_mov_b32_e32 v52, s53
+; SI-NEXT: v_mov_b32_e32 v57, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v56, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v47, s30
+; SI-NEXT: v_mov_b32_e32 v46, s31
+; SI-NEXT: v_mov_b32_e32 v45, s34
+; SI-NEXT: v_mov_b32_e32 v44, s35
+; SI-NEXT: v_mov_b32_e32 v43, s36
+; SI-NEXT: v_mov_b32_e32 v42, s37
+; SI-NEXT: v_mov_b32_e32 v41, s38
+; SI-NEXT: v_mov_b32_e32 v40, s39
+; SI-NEXT: v_mov_b32_e32 v55, s48
+; SI-NEXT: v_mov_b32_e32 v54, s49
+; SI-NEXT: v_mov_b32_e32 v53, s50
+; SI-NEXT: v_mov_b32_e32 v52, s51
; SI-NEXT: v_mov_b32_e32 v28, s42
; SI-NEXT: v_mov_b32_e32 v29, s44
; SI-NEXT: v_mov_b32_e32 v30, s46
@@ -15566,7 +15836,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v48, s88
; SI-NEXT: v_mov_b32_e32 v49, s90
; SI-NEXT: v_mov_b32_e32 v50, s92
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -15649,7 +15919,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v52
-; SI-NEXT: v_readlane_b32 s30, v58, 12
+; SI-NEXT: v_readlane_b32 s30, v58, 10
; SI-NEXT: v_or_b32_e32 v5, v5, v39
; SI-NEXT: v_or_b32_e32 v7, v7, v38
; SI-NEXT: v_or_b32_e32 v9, v9, v37
@@ -15662,9 +15932,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v23, v23, v30
; SI-NEXT: v_or_b32_e32 v25, v25, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: v_readlane_b32 s31, v58, 13
-; SI-NEXT: v_readlane_b32 s53, v58, 11
-; SI-NEXT: v_readlane_b32 s52, v58, 10
+; SI-NEXT: v_readlane_b32 s31, v58, 11
; SI-NEXT: v_readlane_b32 s51, v58, 9
; SI-NEXT: v_readlane_b32 s50, v58, 8
; SI-NEXT: v_readlane_b32 s49, v58, 7
@@ -15695,10 +15963,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
-; VI-NEXT: v_writelane_b32 v56, s34, 0
-; VI-NEXT: v_writelane_b32 v56, s35, 1
-; VI-NEXT: v_writelane_b32 v56, s30, 2
-; VI-NEXT: v_writelane_b32 v56, s31, 3
+; VI-NEXT: v_writelane_b32 v56, s30, 0
+; VI-NEXT: v_writelane_b32 v56, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -15715,7 +15981,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -15741,12 +16007,48 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v27, s6, 1.0
; VI-NEXT: v_add_f32_e64 v26, s7, 1.0
; VI-NEXT: v_add_f32_e64 v25, s8, 1.0
@@ -15803,38 +16105,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v46, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -15863,10 +16135,10 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_mov_b32_e32 v47, s35
-; VI-NEXT: v_mov_b32_e32 v46, s34
-; VI-NEXT: v_mov_b32_e32 v45, s31
-; VI-NEXT: v_mov_b32_e32 v44, s30
+; VI-NEXT: v_mov_b32_e32 v47, s31
+; VI-NEXT: v_mov_b32_e32 v46, s30
+; VI-NEXT: v_mov_b32_e32 v45, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v44, vcc_lo
; VI-NEXT: v_mov_b32_e32 v43, s91
; VI-NEXT: v_mov_b32_e32 v42, s90
; VI-NEXT: v_mov_b32_e32 v41, s89
@@ -15891,7 +16163,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s46
; VI-NEXT: v_mov_b32_e32 v29, s45
; VI-NEXT: v_mov_b32_e32 v28, s44
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
; VI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
; VI-NEXT: v_lshlrev_b32_e32 v45, 16, v45
@@ -15936,7 +16208,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; VI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; VI-NEXT: v_readlane_b32 s30, v56, 2
+; VI-NEXT: v_readlane_b32 s30, v56, 0
; VI-NEXT: v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -15957,9 +16229,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v56, 3
-; VI-NEXT: v_readlane_b32 s35, v56, 1
-; VI-NEXT: v_readlane_b32 s34, v56, 0
+; VI-NEXT: v_readlane_b32 s31, v56, 1
; VI-NEXT: s_or_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -15993,7 +16263,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -16023,8 +16293,44 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v27, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v26, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v25, s8, 1.0
@@ -16081,38 +16387,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v45, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -16169,7 +16445,7 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -16252,11 +16528,11 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s8, 16
@@ -16283,9 +16559,44 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s88, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s89, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v27, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v26, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v25, s6, 1.0
@@ -16342,38 +16653,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -16401,8 +16682,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s15 :: v_dual_mov_b32 v28, s14
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v71.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v70.l
@@ -16449,11 +16730,11 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s8, 16
@@ -16480,9 +16761,44 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s88, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s89, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v23, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v24, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v25, s6, 1.0
@@ -16539,38 +16855,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v70, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v5, s17
@@ -16598,8 +16884,8 @@ define inreg <56 x i16> @bitcast_v28f32_to_v56i16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s15 :: v_dual_mov_b32 v28, s14
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v80, 0xffff, v0
@@ -17955,10 +18241,8 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v28, s83, 25
; SI-NEXT: v_writelane_b32 v28, s84, 26
; SI-NEXT: v_writelane_b32 v28, s85, 27
-; SI-NEXT: v_writelane_b32 v28, s86, 28
-; SI-NEXT: v_writelane_b32 v28, s87, 29
-; SI-NEXT: v_writelane_b32 v28, s30, 30
-; SI-NEXT: v_writelane_b32 v28, s31, 31
+; SI-NEXT: v_writelane_b32 v28, s30, 28
+; SI-NEXT: v_writelane_b32 v28, s31, 29
; SI-NEXT: v_readfirstlane_b32 s7, v13
; SI-NEXT: v_readfirstlane_b32 s9, v12
; SI-NEXT: v_readfirstlane_b32 s11, v11
@@ -17970,23 +18254,23 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s88, v5
; SI-NEXT: v_readfirstlane_b32 s91, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s69, v1
-; SI-NEXT: v_readfirstlane_b32 s80, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; SI-NEXT: v_readfirstlane_b32 s35, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
; SI-NEXT: s_lshr_b32 s78, s29, 16
; SI-NEXT: s_lshr_b32 s89, s28, 16
; SI-NEXT: s_lshr_b32 s92, s27, 16
; SI-NEXT: s_lshr_b32 s95, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s68, s24, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -17998,42 +18282,42 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s79, s88, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s69, 16
-; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s35, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -18047,14 +18331,14 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -18089,56 +18373,65 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -18160,20 +18453,20 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -18231,8 +18524,8 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s63, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v28, 30
+; SI-NEXT: .LBB31_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v28, 28
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -18261,9 +18554,7 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v25, s61
; SI-NEXT: v_mov_b32_e32 v26, s62
; SI-NEXT: v_mov_b32_e32 v27, s63
-; SI-NEXT: v_readlane_b32 s31, v28, 31
-; SI-NEXT: v_readlane_b32 s87, v28, 29
-; SI-NEXT: v_readlane_b32 s86, v28, 28
+; SI-NEXT: v_readlane_b32 s31, v28, 29
; SI-NEXT: v_readlane_b32 s85, v28, 27
; SI-NEXT: v_readlane_b32 s84, v28, 26
; SI-NEXT: v_readlane_b32 s83, v28, 25
@@ -18297,9 +18588,6 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v56i16_to_v28f32_scalar:
; VI: ; %bb.0:
@@ -18340,54 +18628,54 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v28, s87, 29
; VI-NEXT: v_writelane_b32 v28, s30, 30
; VI-NEXT: v_writelane_b32 v28, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s86, v13
-; VI-NEXT: v_readfirstlane_b32 s6, v12
+; VI-NEXT: v_readfirstlane_b32 s81, v13
+; VI-NEXT: v_readfirstlane_b32 s83, v12
; VI-NEXT: v_readfirstlane_b32 s9, v11
-; VI-NEXT: v_readfirstlane_b32 s8, v10
-; VI-NEXT: v_readfirstlane_b32 s87, v9
-; VI-NEXT: v_readfirstlane_b32 s12, v8
+; VI-NEXT: v_readfirstlane_b32 s84, v10
+; VI-NEXT: v_readfirstlane_b32 s85, v9
+; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s73, v7
-; VI-NEXT: v_readfirstlane_b32 s77, v6
-; VI-NEXT: v_readfirstlane_b32 s88, v5
+; VI-NEXT: v_readfirstlane_b32 s75, v6
+; VI-NEXT: v_readfirstlane_b32 s77, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s81, v1
-; VI-NEXT: v_readfirstlane_b32 s84, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s71, v1
+; VI-NEXT: v_readfirstlane_b32 s82, v0
; VI-NEXT: s_lshr_b32 s79, s29, 16
; VI-NEXT: s_lshr_b32 s90, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s67, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s80, s24, 16
-; VI-NEXT: s_lshr_b32 s82, s23, 16
-; VI-NEXT: s_lshr_b32 s85, s22, 16
-; VI-NEXT: s_lshr_b32 s7, s21, 16
-; VI-NEXT: s_lshr_b32 s11, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s69, s23, 16
+; VI-NEXT: s_lshr_b32 s80, s22, 16
+; VI-NEXT: s_lshr_b32 s86, s21, 16
+; VI-NEXT: s_lshr_b32 s87, s20, 16
; VI-NEXT: s_lshr_b32 s13, s19, 16
; VI-NEXT: s_lshr_b32 s15, s18, 16
-; VI-NEXT: s_lshr_b32 s10, s17, 16
-; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_lshr_b32 s64, s86, 16
-; VI-NEXT: s_lshr_b32 s65, s6, 16
-; VI-NEXT: s_lshr_b32 s66, s9, 16
-; VI-NEXT: s_lshr_b32 s14, s8, 16
-; VI-NEXT: s_lshr_b32 s72, s87, 16
-; VI-NEXT: s_lshr_b32 s74, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: s_lshr_b32 s7, s16, 16
+; VI-NEXT: s_lshr_b32 s64, s81, 16
+; VI-NEXT: s_lshr_b32 s10, s83, 16
+; VI-NEXT: s_lshr_b32 s12, s9, 16
+; VI-NEXT: s_lshr_b32 s14, s84, 16
+; VI-NEXT: s_lshr_b32 s72, s85, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
; VI-NEXT: s_lshr_b32 s76, s73, 16
-; VI-NEXT: s_lshr_b32 s78, s77, 16
-; VI-NEXT: s_lshr_b32 s89, s88, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s68, s69, 16
-; VI-NEXT: s_lshr_b32 s71, s81, 16
-; VI-NEXT: s_lshr_b32 s83, s84, 16
+; VI-NEXT: s_lshr_b32 s78, s75, 16
+; VI-NEXT: s_lshr_b32 s89, s77, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s65, s35, 16
+; VI-NEXT: s_lshr_b32 s67, s71, 16
+; VI-NEXT: s_lshr_b32 s70, s82, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
; VI-NEXT: s_lshl_b32 s5, s15, 16
@@ -18395,32 +18683,32 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s19
; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s10, 16
+; VI-NEXT: s_lshl_b32 s41, s88, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -18428,58 +18716,146 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s84
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s82
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s81
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s71
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s88
+; VI-NEXT: s_and_b32 s4, 0xffff, s77
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s77
+; VI-NEXT: s_and_b32 s4, 0xffff, s75
; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s73
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_and_b32 s4, 0xffff, s85
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_and_b32 s4, 0xffff, s84
; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: v_writelane_b32 v29, s26, 0
+; VI-NEXT: s_mov_b32 s26, s27
+; VI-NEXT: s_mov_b32 s27, s77
+; VI-NEXT: v_writelane_b32 v29, s91, 1
+; VI-NEXT: s_mov_b32 s91, s88
+; VI-NEXT: s_mov_b32 s88, s28
+; VI-NEXT: s_mov_b32 s28, s75
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s81
+; VI-NEXT: s_mov_b32 s81, s68
+; VI-NEXT: s_mov_b32 s68, s34
+; VI-NEXT: s_mov_b32 s34, s31
+; VI-NEXT: s_mov_b32 s31, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s90
+; VI-NEXT: s_mov_b32 s90, s89
+; VI-NEXT: s_mov_b32 s89, s79
+; VI-NEXT: s_mov_b32 s79, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s72
+; VI-NEXT: s_mov_b32 s72, s14
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s10, s64
+; VI-NEXT: s_mov_b32 s9, s83
+; VI-NEXT: s_mov_b32 s83, s69
+; VI-NEXT: s_mov_b32 s69, s65
+; VI-NEXT: s_mov_b32 s6, s13
+; VI-NEXT: s_mov_b32 s13, s84
+; VI-NEXT: s_mov_b32 s84, s70
+; VI-NEXT: s_mov_b32 s70, s66
+; VI-NEXT: s_mov_b32 s75, s73
+; VI-NEXT: s_mov_b32 s73, s8
+; VI-NEXT: s_mov_b32 s8, s15
+; VI-NEXT: s_mov_b32 s15, s85
+; VI-NEXT: s_mov_b32 s85, s80
+; VI-NEXT: s_mov_b32 s80, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s80
+; VI-NEXT: s_mov_b32 s80, s85
+; VI-NEXT: s_mov_b32 s85, s15
+; VI-NEXT: s_mov_b32 s15, s8
+; VI-NEXT: s_mov_b32 s8, s73
+; VI-NEXT: s_mov_b32 s73, s75
+; VI-NEXT: s_mov_b32 s66, s70
+; VI-NEXT: s_mov_b32 s70, s84
+; VI-NEXT: s_mov_b32 s84, s13
+; VI-NEXT: s_mov_b32 s13, s6
+; VI-NEXT: s_mov_b32 s65, s69
+; VI-NEXT: s_mov_b32 s69, s83
+; VI-NEXT: s_mov_b32 s83, s9
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s64, s10
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s14, s72
+; VI-NEXT: s_mov_b32 s72, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s79
+; VI-NEXT: s_mov_b32 s79, s89
+; VI-NEXT: s_mov_b32 s89, s90
+; VI-NEXT: s_mov_b32 s90, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s31
+; VI-NEXT: s_mov_b32 s31, s34
+; VI-NEXT: s_mov_b32 s34, s68
+; VI-NEXT: s_mov_b32 s68, s81
+; VI-NEXT: s_mov_b32 s81, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s75, s28
+; VI-NEXT: s_mov_b32 s28, s88
+; VI-NEXT: s_mov_b32 s88, s91
+; VI-NEXT: v_readlane_b32 s91, v29, 1
+; VI-NEXT: s_mov_b32 s77, s27
+; VI-NEXT: s_mov_b32 s27, s26
+; VI-NEXT: v_readlane_b32 s26, v29, 0
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s10, s10, 16
-; VI-NEXT: s_or_b32 s5, s10, s5
+; VI-NEXT: s_lshl_b32 s6, s88, 16
+; VI-NEXT: s_or_b32 s5, s6, s5
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_add_i32 s37, s5, 0x30000
@@ -18494,42 +18870,42 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -18541,37 +18917,37 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s84, s84, 3
+; VI-NEXT: s_add_i32 s82, s82, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s84, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, s82, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s81, s81, 3
+; VI-NEXT: s_add_i32 s71, s71, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s81, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, s71, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s69, s69, 3
+; VI-NEXT: s_add_i32 s35, s35, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s69, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, s35, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s91, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s88, s88, 3
+; VI-NEXT: s_add_i32 s88, s77, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s88, 0xffff
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s77, s77, 3
+; VI-NEXT: s_add_i32 s77, s75, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
; VI-NEXT: s_lshl_b32 s5, s78, 16
@@ -18581,17 +18957,17 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s75, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s12, 3
+; VI-NEXT: s_add_i32 s73, s8, 3
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s87, 3
+; VI-NEXT: s_add_i32 s15, s85, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s8, 3
+; VI-NEXT: s_add_i32 s13, s84, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
; VI-NEXT: s_lshl_b32 s5, s14, 16
@@ -18599,20 +18975,20 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_add_i32 s11, s9, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s6, 3
+; VI-NEXT: s_add_i32 s9, s83, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s86, 3
+; VI-NEXT: s_add_i32 s7, s81, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s63, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v28, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -18679,96 +19055,6 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
-; VI-NEXT: s_mov_b32 vcc_lo, s64
-; VI-NEXT: v_writelane_b32 v29, s75, 0
-; VI-NEXT: s_mov_b32 vcc_hi, s65
-; VI-NEXT: s_mov_b32 s75, s73
-; VI-NEXT: s_mov_b32 s73, s12
-; VI-NEXT: v_writelane_b32 v29, s25, 1
-; VI-NEXT: s_mov_b32 s25, s91
-; VI-NEXT: s_mov_b32 s91, s26
-; VI-NEXT: s_mov_b32 s26, s27
-; VI-NEXT: s_mov_b32 s27, s88
-; VI-NEXT: s_mov_b32 s88, s28
-; VI-NEXT: s_mov_b32 s28, s77
-; VI-NEXT: s_mov_b32 s77, s29
-; VI-NEXT: s_mov_b32 s29, s15
-; VI-NEXT: s_mov_b32 s15, s87
-; VI-NEXT: s_mov_b32 s87, s85
-; VI-NEXT: s_mov_b32 s85, s82
-; VI-NEXT: s_mov_b32 s82, s71
-; VI-NEXT: s_mov_b32 s71, s68
-; VI-NEXT: s_mov_b32 s68, s35
-; VI-NEXT: s_mov_b32 s35, s31
-; VI-NEXT: s_mov_b32 s31, s30
-; VI-NEXT: s_mov_b32 s30, s90
-; VI-NEXT: s_mov_b32 s90, s89
-; VI-NEXT: s_mov_b32 s89, s79
-; VI-NEXT: s_mov_b32 s79, s78
-; VI-NEXT: s_mov_b32 s78, s76
-; VI-NEXT: s_mov_b32 s76, s74
-; VI-NEXT: s_mov_b32 s74, s72
-; VI-NEXT: s_mov_b32 s72, s14
-; VI-NEXT: s_mov_b32 s14, s66
-; VI-NEXT: s_mov_b32 s12, s10
-; VI-NEXT: s_mov_b32 s10, s13
-; VI-NEXT: s_mov_b32 s13, s8
-; VI-NEXT: s_mov_b32 s8, s11
-; VI-NEXT: s_mov_b32 s11, s9
-; VI-NEXT: s_mov_b32 s9, s6
-; VI-NEXT: s_mov_b32 s6, s7
-; VI-NEXT: s_mov_b32 s7, s86
-; VI-NEXT: s_mov_b32 s86, s83
-; VI-NEXT: s_mov_b32 s83, s80
-; VI-NEXT: s_mov_b32 s80, s70
-; VI-NEXT: s_mov_b32 s70, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, s70
-; VI-NEXT: s_mov_b32 s70, s80
-; VI-NEXT: s_mov_b32 s80, s83
-; VI-NEXT: s_mov_b32 s83, s86
-; VI-NEXT: s_mov_b32 s86, s7
-; VI-NEXT: s_mov_b32 s7, s6
-; VI-NEXT: s_mov_b32 s6, s9
-; VI-NEXT: s_mov_b32 s9, s11
-; VI-NEXT: s_mov_b32 s11, s8
-; VI-NEXT: s_mov_b32 s8, s13
-; VI-NEXT: s_mov_b32 s13, s10
-; VI-NEXT: s_mov_b32 s10, s12
-; VI-NEXT: s_mov_b32 s66, s14
-; VI-NEXT: s_mov_b32 s14, s72
-; VI-NEXT: s_mov_b32 s72, s74
-; VI-NEXT: s_mov_b32 s74, s76
-; VI-NEXT: s_mov_b32 s76, s78
-; VI-NEXT: s_mov_b32 s78, s79
-; VI-NEXT: s_mov_b32 s79, s89
-; VI-NEXT: s_mov_b32 s89, s90
-; VI-NEXT: s_mov_b32 s90, s30
-; VI-NEXT: s_mov_b32 s30, s31
-; VI-NEXT: s_mov_b32 s31, s35
-; VI-NEXT: s_mov_b32 s35, s68
-; VI-NEXT: s_mov_b32 s68, s71
-; VI-NEXT: s_mov_b32 s71, s82
-; VI-NEXT: s_mov_b32 s82, s85
-; VI-NEXT: s_mov_b32 s85, s87
-; VI-NEXT: s_mov_b32 s87, s15
-; VI-NEXT: s_mov_b32 s15, s29
-; VI-NEXT: s_mov_b32 s29, s77
-; VI-NEXT: s_mov_b32 s77, s28
-; VI-NEXT: s_mov_b32 s28, s88
-; VI-NEXT: s_mov_b32 s88, s27
-; VI-NEXT: s_mov_b32 s27, s26
-; VI-NEXT: s_mov_b32 s26, s91
-; VI-NEXT: s_mov_b32 s91, s25
-; VI-NEXT: v_readlane_b32 s25, v29, 1
-; VI-NEXT: s_mov_b32 s12, s73
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: v_readlane_b32 s75, v29, 0
-; VI-NEXT: s_mov_b32 s65, vcc_hi
-; VI-NEXT: s_mov_b32 s64, vcc_lo
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v56i16_to_v28f32_scalar:
; GFX9: ; %bb.0:
@@ -18860,10 +19146,18 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -18892,10 +19186,8 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v26, s62, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -18928,7 +19220,7 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -19019,11 +19311,16 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -19053,8 +19350,6 @@ define inreg <28 x float> @bitcast_v56i16_to_v28f32_scalar(<56 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v26, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v27, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -19925,10 +20220,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_writelane_b32 v58, s49, 7
; SI-NEXT: v_writelane_b32 v58, s50, 8
; SI-NEXT: v_writelane_b32 v58, s51, 9
-; SI-NEXT: v_writelane_b32 v58, s52, 10
-; SI-NEXT: v_writelane_b32 v58, s53, 11
-; SI-NEXT: v_writelane_b32 v58, s30, 12
-; SI-NEXT: v_writelane_b32 v58, s31, 13
+; SI-NEXT: v_writelane_b32 v58, s30, 10
+; SI-NEXT: v_writelane_b32 v58, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -19945,22 +20238,22 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s53, s5, 16
-; SI-NEXT: s_lshr_b32 s52, s7, 16
-; SI-NEXT: s_lshr_b32 s51, s9, 16
-; SI-NEXT: s_lshr_b32 s50, s11, 16
-; SI-NEXT: s_lshr_b32 s49, s13, 16
-; SI-NEXT: s_lshr_b32 s48, s15, 16
-; SI-NEXT: s_lshr_b32 s39, s41, 16
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s51, s5, 16
+; SI-NEXT: s_lshr_b32 s50, s7, 16
+; SI-NEXT: s_lshr_b32 s49, s9, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s39, s13, 16
+; SI-NEXT: s_lshr_b32 s38, s15, 16
+; SI-NEXT: s_lshr_b32 s37, s41, 16
+; SI-NEXT: s_lshr_b32 s36, s29, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s23, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -19975,8 +20268,44 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v27, s5, 1.0
; SI-NEXT: v_add_f32_e64 v26, s4, 1.0
; SI-NEXT: v_add_f32_e64 v25, s7, 1.0
@@ -20040,38 +20369,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -20101,20 +20400,20 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v57, s30
-; SI-NEXT: v_mov_b32_e32 v56, s31
-; SI-NEXT: v_mov_b32_e32 v47, s34
-; SI-NEXT: v_mov_b32_e32 v46, s35
-; SI-NEXT: v_mov_b32_e32 v45, s36
-; SI-NEXT: v_mov_b32_e32 v44, s37
-; SI-NEXT: v_mov_b32_e32 v43, s38
-; SI-NEXT: v_mov_b32_e32 v42, s39
-; SI-NEXT: v_mov_b32_e32 v41, s48
-; SI-NEXT: v_mov_b32_e32 v40, s49
-; SI-NEXT: v_mov_b32_e32 v55, s50
-; SI-NEXT: v_mov_b32_e32 v54, s51
-; SI-NEXT: v_mov_b32_e32 v53, s52
-; SI-NEXT: v_mov_b32_e32 v52, s53
+; SI-NEXT: v_mov_b32_e32 v57, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v56, vcc_hi
+; SI-NEXT: v_mov_b32_e32 v47, s30
+; SI-NEXT: v_mov_b32_e32 v46, s31
+; SI-NEXT: v_mov_b32_e32 v45, s34
+; SI-NEXT: v_mov_b32_e32 v44, s35
+; SI-NEXT: v_mov_b32_e32 v43, s36
+; SI-NEXT: v_mov_b32_e32 v42, s37
+; SI-NEXT: v_mov_b32_e32 v41, s38
+; SI-NEXT: v_mov_b32_e32 v40, s39
+; SI-NEXT: v_mov_b32_e32 v55, s48
+; SI-NEXT: v_mov_b32_e32 v54, s49
+; SI-NEXT: v_mov_b32_e32 v53, s50
+; SI-NEXT: v_mov_b32_e32 v52, s51
; SI-NEXT: v_mov_b32_e32 v28, s42
; SI-NEXT: v_mov_b32_e32 v29, s44
; SI-NEXT: v_mov_b32_e32 v30, s46
@@ -20129,7 +20428,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v48, s88
; SI-NEXT: v_mov_b32_e32 v49, s90
; SI-NEXT: v_mov_b32_e32 v50, s92
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -20212,7 +20511,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v52
-; SI-NEXT: v_readlane_b32 s30, v58, 12
+; SI-NEXT: v_readlane_b32 s30, v58, 10
; SI-NEXT: v_or_b32_e32 v5, v5, v39
; SI-NEXT: v_or_b32_e32 v7, v7, v38
; SI-NEXT: v_or_b32_e32 v9, v9, v37
@@ -20225,9 +20524,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; SI-NEXT: v_or_b32_e32 v23, v23, v30
; SI-NEXT: v_or_b32_e32 v25, v25, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: v_readlane_b32 s31, v58, 13
-; SI-NEXT: v_readlane_b32 s53, v58, 11
-; SI-NEXT: v_readlane_b32 s52, v58, 10
+; SI-NEXT: v_readlane_b32 s31, v58, 11
; SI-NEXT: v_readlane_b32 s51, v58, 9
; SI-NEXT: v_readlane_b32 s50, v58, 8
; SI-NEXT: v_readlane_b32 s49, v58, 7
@@ -20258,10 +20555,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
-; VI-NEXT: v_writelane_b32 v56, s34, 0
-; VI-NEXT: v_writelane_b32 v56, s35, 1
-; VI-NEXT: v_writelane_b32 v56, s30, 2
-; VI-NEXT: v_writelane_b32 v56, s31, 3
+; VI-NEXT: v_writelane_b32 v56, s30, 0
+; VI-NEXT: v_writelane_b32 v56, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -20278,7 +20573,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -20304,12 +20599,48 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v27, s6, 1.0
; VI-NEXT: v_add_f32_e64 v26, s7, 1.0
; VI-NEXT: v_add_f32_e64 v25, s8, 1.0
@@ -20366,38 +20697,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v46, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -20426,10 +20727,10 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_mov_b32_e32 v47, s35
-; VI-NEXT: v_mov_b32_e32 v46, s34
-; VI-NEXT: v_mov_b32_e32 v45, s31
-; VI-NEXT: v_mov_b32_e32 v44, s30
+; VI-NEXT: v_mov_b32_e32 v47, s31
+; VI-NEXT: v_mov_b32_e32 v46, s30
+; VI-NEXT: v_mov_b32_e32 v45, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v44, vcc_lo
; VI-NEXT: v_mov_b32_e32 v43, s91
; VI-NEXT: v_mov_b32_e32 v42, s90
; VI-NEXT: v_mov_b32_e32 v41, s89
@@ -20454,7 +20755,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v30, s46
; VI-NEXT: v_mov_b32_e32 v29, s45
; VI-NEXT: v_mov_b32_e32 v28, s44
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
; VI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
; VI-NEXT: v_lshlrev_b32_e32 v45, 16, v45
@@ -20499,7 +20800,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; VI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; VI-NEXT: v_readlane_b32 s30, v56, 2
+; VI-NEXT: v_readlane_b32 s30, v56, 0
; VI-NEXT: v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -20520,9 +20821,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; VI-NEXT: v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v56, 3
-; VI-NEXT: v_readlane_b32 s35, v56, 1
-; VI-NEXT: v_readlane_b32 s34, v56, 0
+; VI-NEXT: v_readlane_b32 s31, v56, 1
; VI-NEXT: s_or_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -20556,7 +20855,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -20586,8 +20885,44 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v27, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v26, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v25, s8, 1.0
@@ -20644,38 +20979,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v45, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -20732,7 +21037,7 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v30, s46
; GFX9-NEXT: v_mov_b32_e32 v29, s45
; GFX9-NEXT: v_mov_b32_e32 v28, s44
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -20815,11 +21120,11 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s8, 16
@@ -20846,9 +21151,44 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s88, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s89, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v27, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v26, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v25, s6, 1.0
@@ -20905,38 +21245,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -20964,8 +21274,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s15 :: v_dual_mov_b32 v28, s14
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v71.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v70.l
@@ -21012,11 +21322,11 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s8, 16
@@ -21043,9 +21353,44 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s88, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s89, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v23, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v24, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v25, s6, 1.0
@@ -21102,38 +21447,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v70, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, s16 :: v_dual_mov_b32 v5, s17
@@ -21161,8 +21476,8 @@ define inreg <56 x half> @bitcast_v28f32_to_v56f16_scalar(<28 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s40 :: v_dual_mov_b32 v28, s15
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v29, s15 :: v_dual_mov_b32 v28, s14
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v80, 0xffff, v0
@@ -22655,10 +22970,8 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s83, 25
; SI-NEXT: v_writelane_b32 v32, s84, 26
; SI-NEXT: v_writelane_b32 v32, s85, 27
-; SI-NEXT: v_writelane_b32 v32, s86, 28
-; SI-NEXT: v_writelane_b32 v32, s87, 29
-; SI-NEXT: v_writelane_b32 v32, s30, 30
-; SI-NEXT: v_writelane_b32 v32, s31, 31
+; SI-NEXT: v_writelane_b32 v32, s30, 28
+; SI-NEXT: v_writelane_b32 v32, s31, 29
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: v_readfirstlane_b32 s8, v12
; SI-NEXT: v_readfirstlane_b32 s10, v11
@@ -22672,21 +22985,21 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s92, v2
; SI-NEXT: v_readfirstlane_b32 s95, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
+; SI-NEXT: v_readfirstlane_b32 s30, v0
; SI-NEXT: s_lshr_b32 s94, s29, 16
-; SI-NEXT: s_lshr_b32 s30, s28, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s70, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s80, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s28, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s68, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s70, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -22699,59 +23012,59 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s89, s88, 16
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
-; SI-NEXT: s_lshr_b32 s31, s95, 16
-; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s95, 16
+; SI-NEXT: s_lshr_b32 s34, s30, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s50, s4, s5
; SI-NEXT: s_and_b32 s4, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s92, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -22789,11 +23102,20 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s87
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s85
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s84
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -22803,14 +23125,14 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s82
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -22820,7 +23142,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s81
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -22829,11 +23151,11 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s80
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_cvt_f32_f16_e32 v6, s21
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s71
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
@@ -22844,7 +23166,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s70
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
@@ -22853,11 +23175,11 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s69
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: v_cvt_f32_f16_e32 v9, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s68
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
@@ -22868,7 +23190,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s35
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
@@ -22877,11 +23199,11 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: v_cvt_f32_f16_e32 v12, s27
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_lo
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
@@ -22901,11 +23223,11 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_or_b32_e32 v12, v14, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s34
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s34
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v16, vcc_hi
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -23015,11 +23337,8 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v26, v27, v26
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
; SI-NEXT: v_or_b32_e32 v27, v29, v27
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -23052,11 +23371,9 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 30
-; SI-NEXT: v_readlane_b32 s31, v32, 31
-; SI-NEXT: v_readlane_b32 s87, v32, 29
-; SI-NEXT: v_readlane_b32 s86, v32, 28
+; SI-NEXT: .LBB35_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 28
+; SI-NEXT: v_readlane_b32 s31, v32, 29
; SI-NEXT: v_readlane_b32 s85, v32, 27
; SI-NEXT: v_readlane_b32 s84, v32, 26
; SI-NEXT: v_readlane_b32 s83, v32, 25
@@ -23130,12 +23447,9 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
-; VI-NEXT: s_lshr_b32 s15, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v11
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: v_readfirstlane_b32 s6, v13
-; VI-NEXT: s_lshr_b32 s61, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v10
; VI-NEXT: v_readfirstlane_b32 s14, v9
; VI-NEXT: v_readfirstlane_b32 s72, v8
@@ -23143,178 +23457,198 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s76, v6
; VI-NEXT: v_readfirstlane_b32 s79, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s80, v1
-; VI-NEXT: v_readfirstlane_b32 s83, v0
-; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s70, v1
+; VI-NEXT: v_readfirstlane_b32 s81, v0
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s75, s28, 16
+; VI-NEXT: s_lshr_b32 s61, s28, 16
; VI-NEXT: s_lshr_b32 s90, s27, 16
-; VI-NEXT: s_lshr_b32 s31, s26, 16
-; VI-NEXT: s_lshr_b32 s68, s25, 16
-; VI-NEXT: s_lshr_b32 s70, s24, 16
-; VI-NEXT: s_lshr_b32 s81, s23, 16
-; VI-NEXT: s_lshr_b32 s84, s22, 16
-; VI-NEXT: s_lshr_b32 s86, s21, 16
-; VI-NEXT: s_lshr_b32 s87, s20, 16
-; VI-NEXT: s_lshr_b32 s7, s19, 16
-; VI-NEXT: s_lshr_b32 s9, s18, 16
-; VI-NEXT: s_lshr_b32 s11, s17, 16
-; VI-NEXT: s_lshr_b32 s13, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; VI-NEXT: s_lshr_b32 s34, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s71, s23, 16
+; VI-NEXT: s_lshr_b32 s82, s22, 16
+; VI-NEXT: s_lshr_b32 s84, s21, 16
+; VI-NEXT: s_lshr_b32 s85, s20, 16
+; VI-NEXT: s_lshr_b32 s86, s19, 16
+; VI-NEXT: s_lshr_b32 s87, s18, 16
+; VI-NEXT: s_lshr_b32 s7, s17, 16
+; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s63, s6, 16
-; VI-NEXT: s_lshr_b32 s60, s12, 16
+; VI-NEXT: s_lshr_b32 s11, s8, 16
+; VI-NEXT: s_lshr_b32 s13, s10, 16
+; VI-NEXT: s_lshr_b32 s15, s12, 16
; VI-NEXT: s_lshr_b32 s59, s14, 16
; VI-NEXT: s_lshr_b32 s58, s72, 16
; VI-NEXT: s_lshr_b32 s57, s74, 16
-; VI-NEXT: s_lshr_b32 s62, s76, 16
+; VI-NEXT: s_lshr_b32 s60, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s71, s69, 16
-; VI-NEXT: s_lshr_b32 s82, s80, 16
-; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s69, s35, 16
+; VI-NEXT: s_lshr_b32 s80, s70, 16
+; VI-NEXT: s_lshr_b32 s83, s81, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
-; VI-NEXT: v_writelane_b32 v33, s61, 1
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s60, 2
-; VI-NEXT: v_writelane_b32 v33, s59, 3
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_writelane_b32 v33, s59, 1
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s11, 16
+; VI-NEXT: s_lshl_b32 s41, s7, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s61, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s83
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s80
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s70
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
-; VI-NEXT: s_lshl_b32 s5, s62, 16
-; VI-NEXT: s_mov_b32 s77, s56
+; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_mov_b32 s73, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s75, s57
+; VI-NEXT: s_mov_b32 s78, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s72
; VI-NEXT: s_lshl_b32 s5, s58, 16
-; VI-NEXT: s_mov_b32 s78, s58
+; VI-NEXT: s_mov_b32 s77, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s14
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s59, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s12
-; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_mov_b32 s88, s60
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s10
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_mov_b32 s75, s61
; VI-NEXT: s_or_b32 s61, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, s15, 16
-; VI-NEXT: s_mov_b32 s88, s62
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s62, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s63, 16
-; VI-NEXT: s_mov_b32 s15, s63
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b32 s75, s61
+; VI-NEXT: s_mov_b32 s88, s60
+; VI-NEXT: s_mov_b32 s73, s56
+; VI-NEXT: s_mov_b32 s78, s57
+; VI-NEXT: s_mov_b32 s77, s58
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v27, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s13
-; VI-NEXT: v_mov_b32_e32 v2, s11
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_add_f16_sdwa v0, v0, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v27
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v27
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s9
+; VI-NEXT: v_mov_b32_e32 v2, s87
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v27
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_add_f16_sdwa v3, v3, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v27
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s87
+; VI-NEXT: v_mov_b32_e32 v4, s85
; VI-NEXT: v_add_f16_sdwa v4, v4, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v27
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s86
+; VI-NEXT: v_mov_b32_e32 v5, s84
; VI-NEXT: v_add_f16_sdwa v5, v5, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v27
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s84
+; VI-NEXT: v_mov_b32_e32 v6, s82
; VI-NEXT: v_add_f16_sdwa v6, v6, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v27
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s81
+; VI-NEXT: v_mov_b32_e32 v7, s71
; VI-NEXT: v_add_f16_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v27
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s70
+; VI-NEXT: v_mov_b32_e32 v8, s68
; VI-NEXT: v_add_f16_sdwa v8, v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v27
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s68
+; VI-NEXT: v_mov_b32_e32 v9, s34
; VI-NEXT: v_add_f16_sdwa v9, v9, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v27
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s31
+; VI-NEXT: v_mov_b32_e32 v10, vcc_hi
; VI-NEXT: v_add_f16_sdwa v10, v10, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v27
; VI-NEXT: v_or_b32_e32 v10, v11, v10
@@ -23322,31 +23656,31 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v11, v11, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v27
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s73
+; VI-NEXT: v_mov_b32_e32 v12, s75
; VI-NEXT: v_add_f16_sdwa v12, v12, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v27
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s77
+; VI-NEXT: v_mov_b32_e32 v13, s73
; VI-NEXT: v_add_f16_sdwa v13, v13, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v27
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s85
+; VI-NEXT: v_mov_b32_e32 v14, s83
; VI-NEXT: v_add_f16_sdwa v14, v14, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s83, v27
+; VI-NEXT: v_add_f16_e32 v15, s81, v27
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s82
+; VI-NEXT: v_mov_b32_e32 v15, s80
; VI-NEXT: v_add_f16_sdwa v15, v15, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s80, v27
+; VI-NEXT: v_add_f16_e32 v16, s70, v27
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s71
+; VI-NEXT: v_mov_b32_e32 v16, s69
; VI-NEXT: v_add_f16_sdwa v16, v16, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s69, v27
+; VI-NEXT: v_add_f16_e32 v17, s35, v27
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s35
+; VI-NEXT: v_mov_b32_e32 v17, s31
; VI-NEXT: v_add_f16_sdwa v17, v17, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s34, v27
+; VI-NEXT: v_add_f16_e32 v18, s30, v27
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s30
+; VI-NEXT: v_mov_b32_e32 v18, vcc_lo
; VI-NEXT: v_add_f16_sdwa v18, v18, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s91, v27
; VI-NEXT: v_or_b32_e32 v18, v19, v18
@@ -23358,49 +23692,38 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; VI-NEXT: v_add_f16_sdwa v20, v20, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s76, v27
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s75
+; VI-NEXT: v_mov_b32_e32 v21, s78
; VI-NEXT: v_add_f16_sdwa v21, v21, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s74, v27
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s78
+; VI-NEXT: v_mov_b32_e32 v22, s77
; VI-NEXT: v_add_f16_sdwa v22, v22, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s72, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s14, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s12, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v24, v25, v24
-; VI-NEXT: v_mov_b32_e32 v25, s4
+; VI-NEXT: v_mov_b32_e32 v25, s15
; VI-NEXT: v_add_f16_sdwa v25, v25, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s10, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v25, v26, v25
-; VI-NEXT: v_mov_b32_e32 v26, s4
+; VI-NEXT: v_mov_b32_e32 v26, s13
; VI-NEXT: v_add_f16_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s8, v27
; VI-NEXT: v_or_b32_e32 v26, v28, v26
-; VI-NEXT: v_mov_b32_e32 v28, s15
+; VI-NEXT: v_mov_b32_e32 v28, s11
; VI-NEXT: v_add_f16_sdwa v28, v28, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s6, v27
; VI-NEXT: v_or_b32_e32 v27, v27, v28
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s88, s62
-; VI-NEXT: s_mov_b32 s77, s56
-; VI-NEXT: s_mov_b32 s75, s57
-; VI-NEXT: s_mov_b32 s78, s58
-; VI-NEXT: s_mov_b32 s15, s63
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -23433,7 +23756,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
+; VI-NEXT: .LBB35_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -23563,10 +23886,18 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v27, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v27 op_sel_hi:[1,0]
@@ -23596,10 +23927,8 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v25, s61, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v26, s62, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v27, s63, v27 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -23632,7 +23961,7 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -23723,11 +24052,16 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -23757,8 +24091,6 @@ define inreg <28 x float> @bitcast_v56f16_to_v28f32_scalar(<56 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v26, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v27, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -24003,10 +24335,18 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s42, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s43, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -24035,7 +24375,7 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -24065,8 +24405,6 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v26, s7
; SI-NEXT: v_mov_b32_e32 v27, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v14i64_to_v14f64_scalar:
; VI: ; %bb.0:
@@ -24087,10 +24425,18 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -24119,7 +24465,7 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -24149,8 +24495,6 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v14i64_to_v14f64_scalar:
; GFX9: ; %bb.0:
@@ -24171,10 +24515,18 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -24203,7 +24555,7 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -24233,8 +24585,6 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v14i64_to_v14f64_scalar:
; GFX11: ; %bb.0:
@@ -24252,11 +24602,16 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s14, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s14, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s14, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -24285,7 +24640,7 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -24301,8 +24656,6 @@ define inreg <14 x double> @bitcast_v14i64_to_v14f64_scalar(<14 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -24496,10 +24849,18 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -24514,10 +24875,8 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; SI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -24550,7 +24909,7 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s55, v32, 11
; SI-NEXT: v_readlane_b32 s54, v32, 10
; SI-NEXT: v_readlane_b32 s53, v32, 9
@@ -24617,10 +24976,18 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -24635,10 +25002,8 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; VI-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -24671,7 +25036,7 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s55, v32, 11
; VI-NEXT: v_readlane_b32 s54, v32, 10
; VI-NEXT: v_readlane_b32 s53, v32, 9
@@ -24738,10 +25103,18 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -24756,10 +25129,8 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[22:23], s[58:59], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -24792,7 +25163,7 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -24860,11 +25231,16 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -24880,8 +25256,6 @@ define inreg <14 x i64> @bitcast_v14f64_to_v14i64_scalar(<14 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -25802,10 +26176,8 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: v_writelane_b32 v28, s49, 7
; SI-NEXT: v_writelane_b32 v28, s50, 8
; SI-NEXT: v_writelane_b32 v28, s51, 9
-; SI-NEXT: v_writelane_b32 v28, s52, 10
-; SI-NEXT: v_writelane_b32 v28, s53, 11
-; SI-NEXT: v_writelane_b32 v28, s30, 12
-; SI-NEXT: v_writelane_b32 v28, s31, 13
+; SI-NEXT: v_writelane_b32 v28, s30, 10
+; SI-NEXT: v_writelane_b32 v28, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -25822,22 +26194,22 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -25852,8 +26224,44 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -25882,20 +26290,20 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -25910,78 +26318,78 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_lshl_b32 s43, s92, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s43
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s53, 16
+; SI-NEXT: s_lshl_b32 s43, s51, 16
; SI-NEXT: s_or_b32 s17, s17, s43
; SI-NEXT: s_lshl_b32 s43, s90, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s43
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s52, 16
+; SI-NEXT: s_lshl_b32 s43, s50, 16
; SI-NEXT: s_or_b32 s19, s19, s43
; SI-NEXT: s_lshl_b32 s43, s88, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s43
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s51, 16
+; SI-NEXT: s_lshl_b32 s43, s49, 16
; SI-NEXT: s_or_b32 s21, s21, s43
; SI-NEXT: s_lshl_b32 s43, s78, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s43
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s50, 16
+; SI-NEXT: s_lshl_b32 s43, s48, 16
; SI-NEXT: s_or_b32 s23, s23, s43
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_lshl_b32 s43, s76, 16
; SI-NEXT: s_or_b32 s24, s24, s43
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s49, 16
+; SI-NEXT: s_lshl_b32 s43, s39, 16
; SI-NEXT: s_or_b32 s25, s25, s43
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s43, s74, 16
; SI-NEXT: s_or_b32 s26, s26, s43
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s48, 16
+; SI-NEXT: s_lshl_b32 s43, s38, 16
; SI-NEXT: s_or_b32 s27, s27, s43
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s43, s72, 16
; SI-NEXT: s_or_b32 s28, s28, s43
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s39, 16
+; SI-NEXT: s_lshl_b32 s43, s37, 16
; SI-NEXT: s_or_b32 s29, s29, s43
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s43, s62, 16
; SI-NEXT: s_or_b32 s40, s40, s43
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s38, 16
+; SI-NEXT: s_lshl_b32 s43, s36, 16
; SI-NEXT: s_or_b32 s41, s41, s43
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s43, s60, 16
; SI-NEXT: s_or_b32 s14, s14, s43
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s37, 16
+; SI-NEXT: s_lshl_b32 s43, s35, 16
; SI-NEXT: s_or_b32 s15, s15, s43
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s43, s58, 16
; SI-NEXT: s_or_b32 s12, s12, s43
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s36, 16
+; SI-NEXT: s_lshl_b32 s43, s34, 16
; SI-NEXT: s_or_b32 s13, s13, s43
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s43, s56, 16
; SI-NEXT: s_or_b32 s10, s10, s43
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s35, 16
+; SI-NEXT: s_lshl_b32 s43, s31, 16
; SI-NEXT: s_or_b32 s11, s11, s43
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s43, s46, 16
; SI-NEXT: s_or_b32 s8, s8, s43
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s34, 16
+; SI-NEXT: s_lshl_b32 s43, s30, 16
; SI-NEXT: s_or_b32 s9, s9, s43
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s43, s44, 16
@@ -25989,13 +26397,13 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_or_b32 s6, s6, s43
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s31, 16
+; SI-NEXT: s_lshl_b32 s43, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s30, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_lo, 16
; SI-NEXT: s_or_b32 s7, s7, s43
; SI-NEXT: s_or_b32 s5, s5, s42
-; SI-NEXT: v_readlane_b32 s30, v28, 12
+; SI-NEXT: v_readlane_b32 s30, v28, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -26024,9 +26432,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v25, s7
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
-; SI-NEXT: v_readlane_b32 s31, v28, 13
-; SI-NEXT: v_readlane_b32 s53, v28, 11
-; SI-NEXT: v_readlane_b32 s52, v28, 10
+; SI-NEXT: v_readlane_b32 s31, v28, 11
; SI-NEXT: v_readlane_b32 s51, v28, 9
; SI-NEXT: v_readlane_b32 s50, v28, 8
; SI-NEXT: v_readlane_b32 s49, v28, 7
@@ -26042,36 +26448,6 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v14i64_to_v56i16_scalar:
; VI: ; %bb.0:
@@ -26079,10 +26455,8 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v28, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v28, s34, 0
-; VI-NEXT: v_writelane_b32 v28, s35, 1
-; VI-NEXT: v_writelane_b32 v28, s30, 2
-; VI-NEXT: v_writelane_b32 v28, s31, 3
+; VI-NEXT: v_writelane_b32 v28, s30, 0
+; VI-NEXT: v_writelane_b32 v28, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -26099,7 +26473,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -26125,12 +26499,48 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -26183,22 +26593,22 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s34, 16
+; VI-NEXT: s_lshl_b32 s16, s30, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s31, 16
+; VI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s30, 16
+; VI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s91, 16
@@ -26272,7 +26682,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s42, s44, 16
; VI-NEXT: s_or_b32 s6, s6, s42
-; VI-NEXT: v_readlane_b32 s30, v28, 2
+; VI-NEXT: v_readlane_b32 s30, v28, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -26301,44 +26711,12 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_readlane_b32 s31, v28, 3
-; VI-NEXT: v_readlane_b32 s35, v28, 1
-; VI-NEXT: v_readlane_b32 s34, v28, 0
+; VI-NEXT: v_readlane_b32 s31, v28, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v28, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v14i64_to_v56i16_scalar:
; GFX9: ; %bb.0:
@@ -26359,7 +26737,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -26389,8 +26767,44 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -26447,7 +26861,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s93
@@ -26505,36 +26919,6 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v14i64_to_v56i16_scalar:
; GFX11: ; %bb.0:
@@ -26552,7 +26936,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s90, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s4, 16
; GFX11-NEXT: s_lshr_b32 s15, s5, 16
@@ -26582,9 +26966,44 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s90, -1
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -26641,7 +27060,7 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s89
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s88
@@ -26686,36 +27105,6 @@ define inreg <56 x i16> @bitcast_v14i64_to_v56i16_scalar(<14 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -28014,10 +28403,8 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v28, s83, 25
; SI-NEXT: v_writelane_b32 v28, s84, 26
; SI-NEXT: v_writelane_b32 v28, s85, 27
-; SI-NEXT: v_writelane_b32 v28, s86, 28
-; SI-NEXT: v_writelane_b32 v28, s87, 29
-; SI-NEXT: v_writelane_b32 v28, s30, 30
-; SI-NEXT: v_writelane_b32 v28, s31, 31
+; SI-NEXT: v_writelane_b32 v28, s30, 28
+; SI-NEXT: v_writelane_b32 v28, s31, 29
; SI-NEXT: v_readfirstlane_b32 s7, v13
; SI-NEXT: v_readfirstlane_b32 s9, v12
; SI-NEXT: v_readfirstlane_b32 s11, v11
@@ -28029,23 +28416,23 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s88, v5
; SI-NEXT: v_readfirstlane_b32 s91, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s69, v1
-; SI-NEXT: v_readfirstlane_b32 s80, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; SI-NEXT: v_readfirstlane_b32 s35, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
; SI-NEXT: s_lshr_b32 s78, s29, 16
; SI-NEXT: s_lshr_b32 s89, s28, 16
; SI-NEXT: s_lshr_b32 s92, s27, 16
; SI-NEXT: s_lshr_b32 s95, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s68, s24, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -28057,42 +28444,42 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s79, s88, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s69, 16
-; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s35, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -28106,14 +28493,14 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -28148,56 +28535,65 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -28219,20 +28615,20 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -28290,8 +28686,8 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s63, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v28, 30
+; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v28, 28
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -28320,9 +28716,7 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v25, s61
; SI-NEXT: v_mov_b32_e32 v26, s62
; SI-NEXT: v_mov_b32_e32 v27, s63
-; SI-NEXT: v_readlane_b32 s31, v28, 31
-; SI-NEXT: v_readlane_b32 s87, v28, 29
-; SI-NEXT: v_readlane_b32 s86, v28, 28
+; SI-NEXT: v_readlane_b32 s31, v28, 29
; SI-NEXT: v_readlane_b32 s85, v28, 27
; SI-NEXT: v_readlane_b32 s84, v28, 26
; SI-NEXT: v_readlane_b32 s83, v28, 25
@@ -28356,9 +28750,6 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v56i16_to_v14i64_scalar:
; VI: ; %bb.0:
@@ -28399,54 +28790,54 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v28, s87, 29
; VI-NEXT: v_writelane_b32 v28, s30, 30
; VI-NEXT: v_writelane_b32 v28, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s86, v13
-; VI-NEXT: v_readfirstlane_b32 s6, v12
+; VI-NEXT: v_readfirstlane_b32 s81, v13
+; VI-NEXT: v_readfirstlane_b32 s83, v12
; VI-NEXT: v_readfirstlane_b32 s9, v11
-; VI-NEXT: v_readfirstlane_b32 s8, v10
-; VI-NEXT: v_readfirstlane_b32 s87, v9
-; VI-NEXT: v_readfirstlane_b32 s12, v8
+; VI-NEXT: v_readfirstlane_b32 s84, v10
+; VI-NEXT: v_readfirstlane_b32 s85, v9
+; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s73, v7
-; VI-NEXT: v_readfirstlane_b32 s77, v6
-; VI-NEXT: v_readfirstlane_b32 s88, v5
+; VI-NEXT: v_readfirstlane_b32 s75, v6
+; VI-NEXT: v_readfirstlane_b32 s77, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s81, v1
-; VI-NEXT: v_readfirstlane_b32 s84, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s71, v1
+; VI-NEXT: v_readfirstlane_b32 s82, v0
; VI-NEXT: s_lshr_b32 s79, s29, 16
; VI-NEXT: s_lshr_b32 s90, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s67, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s80, s24, 16
-; VI-NEXT: s_lshr_b32 s82, s23, 16
-; VI-NEXT: s_lshr_b32 s85, s22, 16
-; VI-NEXT: s_lshr_b32 s7, s21, 16
-; VI-NEXT: s_lshr_b32 s11, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s69, s23, 16
+; VI-NEXT: s_lshr_b32 s80, s22, 16
+; VI-NEXT: s_lshr_b32 s86, s21, 16
+; VI-NEXT: s_lshr_b32 s87, s20, 16
; VI-NEXT: s_lshr_b32 s13, s19, 16
; VI-NEXT: s_lshr_b32 s15, s18, 16
-; VI-NEXT: s_lshr_b32 s10, s17, 16
-; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_lshr_b32 s64, s86, 16
-; VI-NEXT: s_lshr_b32 s65, s6, 16
-; VI-NEXT: s_lshr_b32 s66, s9, 16
-; VI-NEXT: s_lshr_b32 s14, s8, 16
-; VI-NEXT: s_lshr_b32 s72, s87, 16
-; VI-NEXT: s_lshr_b32 s74, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: s_lshr_b32 s7, s16, 16
+; VI-NEXT: s_lshr_b32 s64, s81, 16
+; VI-NEXT: s_lshr_b32 s10, s83, 16
+; VI-NEXT: s_lshr_b32 s12, s9, 16
+; VI-NEXT: s_lshr_b32 s14, s84, 16
+; VI-NEXT: s_lshr_b32 s72, s85, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
; VI-NEXT: s_lshr_b32 s76, s73, 16
-; VI-NEXT: s_lshr_b32 s78, s77, 16
-; VI-NEXT: s_lshr_b32 s89, s88, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s68, s69, 16
-; VI-NEXT: s_lshr_b32 s71, s81, 16
-; VI-NEXT: s_lshr_b32 s83, s84, 16
+; VI-NEXT: s_lshr_b32 s78, s75, 16
+; VI-NEXT: s_lshr_b32 s89, s77, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s65, s35, 16
+; VI-NEXT: s_lshr_b32 s67, s71, 16
+; VI-NEXT: s_lshr_b32 s70, s82, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
; VI-NEXT: s_lshl_b32 s5, s15, 16
@@ -28454,32 +28845,32 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s19
; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s10, 16
+; VI-NEXT: s_lshl_b32 s41, s88, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -28487,58 +28878,146 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s84
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s82
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s81
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s71
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s88
+; VI-NEXT: s_and_b32 s4, 0xffff, s77
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s77
+; VI-NEXT: s_and_b32 s4, 0xffff, s75
; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s73
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_and_b32 s4, 0xffff, s85
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_and_b32 s4, 0xffff, s84
; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: v_writelane_b32 v29, s26, 0
+; VI-NEXT: s_mov_b32 s26, s27
+; VI-NEXT: s_mov_b32 s27, s77
+; VI-NEXT: v_writelane_b32 v29, s91, 1
+; VI-NEXT: s_mov_b32 s91, s88
+; VI-NEXT: s_mov_b32 s88, s28
+; VI-NEXT: s_mov_b32 s28, s75
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s81
+; VI-NEXT: s_mov_b32 s81, s68
+; VI-NEXT: s_mov_b32 s68, s34
+; VI-NEXT: s_mov_b32 s34, s31
+; VI-NEXT: s_mov_b32 s31, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s90
+; VI-NEXT: s_mov_b32 s90, s89
+; VI-NEXT: s_mov_b32 s89, s79
+; VI-NEXT: s_mov_b32 s79, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s72
+; VI-NEXT: s_mov_b32 s72, s14
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s10, s64
+; VI-NEXT: s_mov_b32 s9, s83
+; VI-NEXT: s_mov_b32 s83, s69
+; VI-NEXT: s_mov_b32 s69, s65
+; VI-NEXT: s_mov_b32 s6, s13
+; VI-NEXT: s_mov_b32 s13, s84
+; VI-NEXT: s_mov_b32 s84, s70
+; VI-NEXT: s_mov_b32 s70, s66
+; VI-NEXT: s_mov_b32 s75, s73
+; VI-NEXT: s_mov_b32 s73, s8
+; VI-NEXT: s_mov_b32 s8, s15
+; VI-NEXT: s_mov_b32 s15, s85
+; VI-NEXT: s_mov_b32 s85, s80
+; VI-NEXT: s_mov_b32 s80, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s80
+; VI-NEXT: s_mov_b32 s80, s85
+; VI-NEXT: s_mov_b32 s85, s15
+; VI-NEXT: s_mov_b32 s15, s8
+; VI-NEXT: s_mov_b32 s8, s73
+; VI-NEXT: s_mov_b32 s73, s75
+; VI-NEXT: s_mov_b32 s66, s70
+; VI-NEXT: s_mov_b32 s70, s84
+; VI-NEXT: s_mov_b32 s84, s13
+; VI-NEXT: s_mov_b32 s13, s6
+; VI-NEXT: s_mov_b32 s65, s69
+; VI-NEXT: s_mov_b32 s69, s83
+; VI-NEXT: s_mov_b32 s83, s9
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s64, s10
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s14, s72
+; VI-NEXT: s_mov_b32 s72, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s79
+; VI-NEXT: s_mov_b32 s79, s89
+; VI-NEXT: s_mov_b32 s89, s90
+; VI-NEXT: s_mov_b32 s90, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s31
+; VI-NEXT: s_mov_b32 s31, s34
+; VI-NEXT: s_mov_b32 s34, s68
+; VI-NEXT: s_mov_b32 s68, s81
+; VI-NEXT: s_mov_b32 s81, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s75, s28
+; VI-NEXT: s_mov_b32 s28, s88
+; VI-NEXT: s_mov_b32 s88, s91
+; VI-NEXT: v_readlane_b32 s91, v29, 1
+; VI-NEXT: s_mov_b32 s77, s27
+; VI-NEXT: s_mov_b32 s27, s26
+; VI-NEXT: v_readlane_b32 s26, v29, 0
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s10, s10, 16
-; VI-NEXT: s_or_b32 s5, s10, s5
+; VI-NEXT: s_lshl_b32 s6, s88, 16
+; VI-NEXT: s_or_b32 s5, s6, s5
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_add_i32 s37, s5, 0x30000
@@ -28553,42 +29032,42 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -28600,37 +29079,37 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s84, s84, 3
+; VI-NEXT: s_add_i32 s82, s82, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s84, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, s82, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s81, s81, 3
+; VI-NEXT: s_add_i32 s71, s71, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s81, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, s71, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s69, s69, 3
+; VI-NEXT: s_add_i32 s35, s35, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s69, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, s35, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s91, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s88, s88, 3
+; VI-NEXT: s_add_i32 s88, s77, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s88, 0xffff
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s77, s77, 3
+; VI-NEXT: s_add_i32 s77, s75, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
; VI-NEXT: s_lshl_b32 s5, s78, 16
@@ -28640,17 +29119,17 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_and_b32 s4, s75, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s12, 3
+; VI-NEXT: s_add_i32 s73, s8, 3
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s87, 3
+; VI-NEXT: s_add_i32 s15, s85, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s8, 3
+; VI-NEXT: s_add_i32 s13, s84, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
; VI-NEXT: s_lshl_b32 s5, s14, 16
@@ -28658,20 +29137,20 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s11, s9, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s6, 3
+; VI-NEXT: s_add_i32 s9, s83, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s86, 3
+; VI-NEXT: s_add_i32 s7, s81, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s63, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v28, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -28738,96 +29217,6 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
-; VI-NEXT: s_mov_b32 vcc_lo, s64
-; VI-NEXT: v_writelane_b32 v29, s75, 0
-; VI-NEXT: s_mov_b32 vcc_hi, s65
-; VI-NEXT: s_mov_b32 s75, s73
-; VI-NEXT: s_mov_b32 s73, s12
-; VI-NEXT: v_writelane_b32 v29, s25, 1
-; VI-NEXT: s_mov_b32 s25, s91
-; VI-NEXT: s_mov_b32 s91, s26
-; VI-NEXT: s_mov_b32 s26, s27
-; VI-NEXT: s_mov_b32 s27, s88
-; VI-NEXT: s_mov_b32 s88, s28
-; VI-NEXT: s_mov_b32 s28, s77
-; VI-NEXT: s_mov_b32 s77, s29
-; VI-NEXT: s_mov_b32 s29, s15
-; VI-NEXT: s_mov_b32 s15, s87
-; VI-NEXT: s_mov_b32 s87, s85
-; VI-NEXT: s_mov_b32 s85, s82
-; VI-NEXT: s_mov_b32 s82, s71
-; VI-NEXT: s_mov_b32 s71, s68
-; VI-NEXT: s_mov_b32 s68, s35
-; VI-NEXT: s_mov_b32 s35, s31
-; VI-NEXT: s_mov_b32 s31, s30
-; VI-NEXT: s_mov_b32 s30, s90
-; VI-NEXT: s_mov_b32 s90, s89
-; VI-NEXT: s_mov_b32 s89, s79
-; VI-NEXT: s_mov_b32 s79, s78
-; VI-NEXT: s_mov_b32 s78, s76
-; VI-NEXT: s_mov_b32 s76, s74
-; VI-NEXT: s_mov_b32 s74, s72
-; VI-NEXT: s_mov_b32 s72, s14
-; VI-NEXT: s_mov_b32 s14, s66
-; VI-NEXT: s_mov_b32 s12, s10
-; VI-NEXT: s_mov_b32 s10, s13
-; VI-NEXT: s_mov_b32 s13, s8
-; VI-NEXT: s_mov_b32 s8, s11
-; VI-NEXT: s_mov_b32 s11, s9
-; VI-NEXT: s_mov_b32 s9, s6
-; VI-NEXT: s_mov_b32 s6, s7
-; VI-NEXT: s_mov_b32 s7, s86
-; VI-NEXT: s_mov_b32 s86, s83
-; VI-NEXT: s_mov_b32 s83, s80
-; VI-NEXT: s_mov_b32 s80, s70
-; VI-NEXT: s_mov_b32 s70, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, s70
-; VI-NEXT: s_mov_b32 s70, s80
-; VI-NEXT: s_mov_b32 s80, s83
-; VI-NEXT: s_mov_b32 s83, s86
-; VI-NEXT: s_mov_b32 s86, s7
-; VI-NEXT: s_mov_b32 s7, s6
-; VI-NEXT: s_mov_b32 s6, s9
-; VI-NEXT: s_mov_b32 s9, s11
-; VI-NEXT: s_mov_b32 s11, s8
-; VI-NEXT: s_mov_b32 s8, s13
-; VI-NEXT: s_mov_b32 s13, s10
-; VI-NEXT: s_mov_b32 s10, s12
-; VI-NEXT: s_mov_b32 s66, s14
-; VI-NEXT: s_mov_b32 s14, s72
-; VI-NEXT: s_mov_b32 s72, s74
-; VI-NEXT: s_mov_b32 s74, s76
-; VI-NEXT: s_mov_b32 s76, s78
-; VI-NEXT: s_mov_b32 s78, s79
-; VI-NEXT: s_mov_b32 s79, s89
-; VI-NEXT: s_mov_b32 s89, s90
-; VI-NEXT: s_mov_b32 s90, s30
-; VI-NEXT: s_mov_b32 s30, s31
-; VI-NEXT: s_mov_b32 s31, s35
-; VI-NEXT: s_mov_b32 s35, s68
-; VI-NEXT: s_mov_b32 s68, s71
-; VI-NEXT: s_mov_b32 s71, s82
-; VI-NEXT: s_mov_b32 s82, s85
-; VI-NEXT: s_mov_b32 s85, s87
-; VI-NEXT: s_mov_b32 s87, s15
-; VI-NEXT: s_mov_b32 s15, s29
-; VI-NEXT: s_mov_b32 s29, s77
-; VI-NEXT: s_mov_b32 s77, s28
-; VI-NEXT: s_mov_b32 s28, s88
-; VI-NEXT: s_mov_b32 s88, s27
-; VI-NEXT: s_mov_b32 s27, s26
-; VI-NEXT: s_mov_b32 s26, s91
-; VI-NEXT: s_mov_b32 s91, s25
-; VI-NEXT: v_readlane_b32 s25, v29, 1
-; VI-NEXT: s_mov_b32 s12, s73
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: v_readlane_b32 s75, v29, 0
-; VI-NEXT: s_mov_b32 s65, vcc_hi
-; VI-NEXT: s_mov_b32 s64, vcc_lo
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v56i16_to_v14i64_scalar:
; GFX9: ; %bb.0:
@@ -28919,10 +29308,18 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -28951,10 +29348,8 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v26, s62, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -28987,7 +29382,7 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -29078,11 +29473,16 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -29112,8 +29512,6 @@ define inreg <14 x i64> @bitcast_v56i16_to_v14i64_scalar(<56 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v26, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v27, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -30017,10 +30415,8 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: v_writelane_b32 v28, s49, 7
; SI-NEXT: v_writelane_b32 v28, s50, 8
; SI-NEXT: v_writelane_b32 v28, s51, 9
-; SI-NEXT: v_writelane_b32 v28, s52, 10
-; SI-NEXT: v_writelane_b32 v28, s53, 11
-; SI-NEXT: v_writelane_b32 v28, s30, 12
-; SI-NEXT: v_writelane_b32 v28, s31, 13
+; SI-NEXT: v_writelane_b32 v28, s30, 10
+; SI-NEXT: v_writelane_b32 v28, s31, 11
; SI-NEXT: v_readfirstlane_b32 s40, v14
; SI-NEXT: v_readfirstlane_b32 s5, v13
; SI-NEXT: v_readfirstlane_b32 s4, v12
@@ -30037,22 +30433,22 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s41, v1
; SI-NEXT: s_cmp_lg_u32 s40, 0
; SI-NEXT: v_readfirstlane_b32 s40, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -30067,8 +30463,44 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -30097,20 +30529,20 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: s_lshr_b32 s30, s5, 16
-; SI-NEXT: s_lshr_b32 s31, s7, 16
-; SI-NEXT: s_lshr_b32 s34, s9, 16
-; SI-NEXT: s_lshr_b32 s35, s11, 16
-; SI-NEXT: s_lshr_b32 s36, s13, 16
-; SI-NEXT: s_lshr_b32 s37, s15, 16
-; SI-NEXT: s_lshr_b32 s38, s41, 16
-; SI-NEXT: s_lshr_b32 s39, s29, 16
-; SI-NEXT: s_lshr_b32 s48, s27, 16
-; SI-NEXT: s_lshr_b32 s49, s25, 16
-; SI-NEXT: s_lshr_b32 s50, s23, 16
-; SI-NEXT: s_lshr_b32 s51, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s19, 16
-; SI-NEXT: s_lshr_b32 s53, s17, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s5, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s7, 16
+; SI-NEXT: s_lshr_b32 s30, s9, 16
+; SI-NEXT: s_lshr_b32 s31, s11, 16
+; SI-NEXT: s_lshr_b32 s34, s13, 16
+; SI-NEXT: s_lshr_b32 s35, s15, 16
+; SI-NEXT: s_lshr_b32 s36, s41, 16
+; SI-NEXT: s_lshr_b32 s37, s29, 16
+; SI-NEXT: s_lshr_b32 s38, s27, 16
+; SI-NEXT: s_lshr_b32 s39, s25, 16
+; SI-NEXT: s_lshr_b32 s48, s23, 16
+; SI-NEXT: s_lshr_b32 s49, s21, 16
+; SI-NEXT: s_lshr_b32 s50, s19, 16
+; SI-NEXT: s_lshr_b32 s51, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
@@ -30125,78 +30557,78 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_lshl_b32 s43, s92, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s43
; SI-NEXT: s_and_b32 s17, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s53, 16
+; SI-NEXT: s_lshl_b32 s43, s51, 16
; SI-NEXT: s_or_b32 s17, s17, s43
; SI-NEXT: s_lshl_b32 s43, s90, 16
; SI-NEXT: s_and_b32 s18, s18, 0xffff
; SI-NEXT: s_or_b32 s18, s18, s43
; SI-NEXT: s_and_b32 s19, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s52, 16
+; SI-NEXT: s_lshl_b32 s43, s50, 16
; SI-NEXT: s_or_b32 s19, s19, s43
; SI-NEXT: s_lshl_b32 s43, s88, 16
; SI-NEXT: s_and_b32 s20, s20, 0xffff
; SI-NEXT: s_or_b32 s20, s20, s43
; SI-NEXT: s_and_b32 s21, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s51, 16
+; SI-NEXT: s_lshl_b32 s43, s49, 16
; SI-NEXT: s_or_b32 s21, s21, s43
; SI-NEXT: s_lshl_b32 s43, s78, 16
; SI-NEXT: s_and_b32 s22, s22, 0xffff
; SI-NEXT: s_or_b32 s22, s22, s43
; SI-NEXT: s_and_b32 s23, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s50, 16
+; SI-NEXT: s_lshl_b32 s43, s48, 16
; SI-NEXT: s_or_b32 s23, s23, s43
; SI-NEXT: s_and_b32 s24, s24, 0xffff
; SI-NEXT: s_lshl_b32 s43, s76, 16
; SI-NEXT: s_or_b32 s24, s24, s43
; SI-NEXT: s_and_b32 s25, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s49, 16
+; SI-NEXT: s_lshl_b32 s43, s39, 16
; SI-NEXT: s_or_b32 s25, s25, s43
; SI-NEXT: s_and_b32 s26, s26, 0xffff
; SI-NEXT: s_lshl_b32 s43, s74, 16
; SI-NEXT: s_or_b32 s26, s26, s43
; SI-NEXT: s_and_b32 s27, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s48, 16
+; SI-NEXT: s_lshl_b32 s43, s38, 16
; SI-NEXT: s_or_b32 s27, s27, s43
; SI-NEXT: s_and_b32 s28, s28, 0xffff
; SI-NEXT: s_lshl_b32 s43, s72, 16
; SI-NEXT: s_or_b32 s28, s28, s43
; SI-NEXT: s_and_b32 s29, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s39, 16
+; SI-NEXT: s_lshl_b32 s43, s37, 16
; SI-NEXT: s_or_b32 s29, s29, s43
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s43, s62, 16
; SI-NEXT: s_or_b32 s40, s40, s43
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s38, 16
+; SI-NEXT: s_lshl_b32 s43, s36, 16
; SI-NEXT: s_or_b32 s41, s41, s43
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s43, s60, 16
; SI-NEXT: s_or_b32 s14, s14, s43
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s37, 16
+; SI-NEXT: s_lshl_b32 s43, s35, 16
; SI-NEXT: s_or_b32 s15, s15, s43
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s43, s58, 16
; SI-NEXT: s_or_b32 s12, s12, s43
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s36, 16
+; SI-NEXT: s_lshl_b32 s43, s34, 16
; SI-NEXT: s_or_b32 s13, s13, s43
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_lshl_b32 s43, s56, 16
; SI-NEXT: s_or_b32 s10, s10, s43
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s35, 16
+; SI-NEXT: s_lshl_b32 s43, s31, 16
; SI-NEXT: s_or_b32 s11, s11, s43
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s43, s46, 16
; SI-NEXT: s_or_b32 s8, s8, s43
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s34, 16
+; SI-NEXT: s_lshl_b32 s43, s30, 16
; SI-NEXT: s_or_b32 s9, s9, s43
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s43, s44, 16
@@ -30204,13 +30636,13 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_or_b32 s6, s6, s43
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s43, s31, 16
+; SI-NEXT: s_lshl_b32 s43, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s4, s42
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s30, 16
+; SI-NEXT: s_lshl_b32 s42, vcc_lo, 16
; SI-NEXT: s_or_b32 s7, s7, s43
; SI-NEXT: s_or_b32 s5, s5, s42
-; SI-NEXT: v_readlane_b32 s30, v28, 12
+; SI-NEXT: v_readlane_b32 s30, v28, 10
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -30239,9 +30671,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v25, s7
; SI-NEXT: v_mov_b32_e32 v26, s4
; SI-NEXT: v_mov_b32_e32 v27, s5
-; SI-NEXT: v_readlane_b32 s31, v28, 13
-; SI-NEXT: v_readlane_b32 s53, v28, 11
-; SI-NEXT: v_readlane_b32 s52, v28, 10
+; SI-NEXT: v_readlane_b32 s31, v28, 11
; SI-NEXT: v_readlane_b32 s51, v28, 9
; SI-NEXT: v_readlane_b32 s50, v28, 8
; SI-NEXT: v_readlane_b32 s49, v28, 7
@@ -30257,36 +30687,6 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v14i64_to_v56f16_scalar:
; VI: ; %bb.0:
@@ -30294,10 +30694,8 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v28, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v28, s34, 0
-; VI-NEXT: v_writelane_b32 v28, s35, 1
-; VI-NEXT: v_writelane_b32 v28, s30, 2
-; VI-NEXT: v_writelane_b32 v28, s31, 3
+; VI-NEXT: v_writelane_b32 v28, s30, 0
+; VI-NEXT: v_writelane_b32 v28, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s7, v12
@@ -30314,7 +30712,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s42, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s43, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s6, 16
; VI-NEXT: s_lshr_b32 s45, s7, 16
@@ -30340,12 +30738,48 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -30398,22 +30832,22 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s22, 16
; VI-NEXT: s_lshr_b32 s90, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s34, 16
+; VI-NEXT: s_lshl_b32 s16, s30, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s31, 16
+; VI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s30, 16
+; VI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s91, 16
@@ -30487,7 +30921,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s42, s44, 16
; VI-NEXT: s_or_b32 s6, s6, s42
-; VI-NEXT: v_readlane_b32 s30, v28, 2
+; VI-NEXT: v_readlane_b32 s30, v28, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -30516,44 +30950,12 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v25, s8
; VI-NEXT: v_mov_b32_e32 v26, s7
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_readlane_b32 s31, v28, 3
-; VI-NEXT: v_readlane_b32 s35, v28, 1
-; VI-NEXT: v_readlane_b32 s34, v28, 0
+; VI-NEXT: v_readlane_b32 s31, v28, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v28, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v14i64_to_v56f16_scalar:
; GFX9: ; %bb.0:
@@ -30574,7 +30976,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s42, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s43, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s6, 16
; GFX9-NEXT: s_lshr_b32 s45, s7, 16
@@ -30604,8 +31006,44 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -30662,7 +31100,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s18, 16
; GFX9-NEXT: s_lshr_b32 s94, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s93
@@ -30720,36 +31158,6 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v26, s7
; GFX9-NEXT: v_mov_b32_e32 v27, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v14i64_to_v56f16_scalar:
; GFX11: ; %bb.0:
@@ -30767,7 +31175,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s13, v0
; GFX11-NEXT: s_cmp_lg_u32 s14, 0
; GFX11-NEXT: s_mov_b32 s90, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s14, s4, 16
; GFX11-NEXT: s_lshr_b32 s15, s5, 16
@@ -30797,9 +31205,44 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s90, -1
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: ; implicit-def: $sgpr15
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -30856,7 +31299,7 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s79, s2, 16
; GFX11-NEXT: s_lshr_b32 s88, s1, 16
; GFX11-NEXT: s_lshr_b32 s89, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s89
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s88
@@ -30901,36 +31344,6 @@ define inreg <56 x half> @bitcast_v14i64_to_v56f16_scalar(<14 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v24, s7 :: v_dual_mov_b32 v25, s6
; GFX11-NEXT: v_dual_mov_b32 v26, s5 :: v_dual_mov_b32 v27, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: ; implicit-def: $sgpr15
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -32366,10 +32779,8 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s83, 25
; SI-NEXT: v_writelane_b32 v32, s84, 26
; SI-NEXT: v_writelane_b32 v32, s85, 27
-; SI-NEXT: v_writelane_b32 v32, s86, 28
-; SI-NEXT: v_writelane_b32 v32, s87, 29
-; SI-NEXT: v_writelane_b32 v32, s30, 30
-; SI-NEXT: v_writelane_b32 v32, s31, 31
+; SI-NEXT: v_writelane_b32 v32, s30, 28
+; SI-NEXT: v_writelane_b32 v32, s31, 29
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: v_readfirstlane_b32 s8, v12
; SI-NEXT: v_readfirstlane_b32 s10, v11
@@ -32383,21 +32794,21 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s92, v2
; SI-NEXT: v_readfirstlane_b32 s95, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
+; SI-NEXT: v_readfirstlane_b32 s30, v0
; SI-NEXT: s_lshr_b32 s94, s29, 16
-; SI-NEXT: s_lshr_b32 s30, s28, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s70, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s80, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s28, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s68, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s70, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -32410,59 +32821,59 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s89, s88, 16
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
-; SI-NEXT: s_lshr_b32 s31, s95, 16
-; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s95, 16
+; SI-NEXT: s_lshr_b32 s34, s30, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s50, s4, s5
; SI-NEXT: s_and_b32 s4, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s92, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -32500,11 +32911,20 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s87
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s85
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s84
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -32514,14 +32934,14 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s82
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -32531,7 +32951,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s81
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -32540,11 +32960,11 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s80
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_cvt_f32_f16_e32 v6, s21
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s71
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
@@ -32555,7 +32975,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s70
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
@@ -32564,11 +32984,11 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s69
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: v_cvt_f32_f16_e32 v9, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s68
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
@@ -32579,7 +32999,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s35
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
@@ -32588,11 +33008,11 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: v_cvt_f32_f16_e32 v12, s27
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_lo
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
@@ -32612,11 +33032,11 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_or_b32_e32 v12, v14, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s34
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s34
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v16, vcc_hi
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -32726,11 +33146,8 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v26, v27, v26
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
; SI-NEXT: v_or_b32_e32 v27, v29, v27
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -32763,11 +33180,9 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 30
-; SI-NEXT: v_readlane_b32 s31, v32, 31
-; SI-NEXT: v_readlane_b32 s87, v32, 29
-; SI-NEXT: v_readlane_b32 s86, v32, 28
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 28
+; SI-NEXT: v_readlane_b32 s31, v32, 29
; SI-NEXT: v_readlane_b32 s85, v32, 27
; SI-NEXT: v_readlane_b32 s84, v32, 26
; SI-NEXT: v_readlane_b32 s83, v32, 25
@@ -32841,12 +33256,9 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
-; VI-NEXT: s_lshr_b32 s15, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v11
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: v_readfirstlane_b32 s6, v13
-; VI-NEXT: s_lshr_b32 s61, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v10
; VI-NEXT: v_readfirstlane_b32 s14, v9
; VI-NEXT: v_readfirstlane_b32 s72, v8
@@ -32854,178 +33266,198 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s76, v6
; VI-NEXT: v_readfirstlane_b32 s79, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s80, v1
-; VI-NEXT: v_readfirstlane_b32 s83, v0
-; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s70, v1
+; VI-NEXT: v_readfirstlane_b32 s81, v0
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s75, s28, 16
+; VI-NEXT: s_lshr_b32 s61, s28, 16
; VI-NEXT: s_lshr_b32 s90, s27, 16
-; VI-NEXT: s_lshr_b32 s31, s26, 16
-; VI-NEXT: s_lshr_b32 s68, s25, 16
-; VI-NEXT: s_lshr_b32 s70, s24, 16
-; VI-NEXT: s_lshr_b32 s81, s23, 16
-; VI-NEXT: s_lshr_b32 s84, s22, 16
-; VI-NEXT: s_lshr_b32 s86, s21, 16
-; VI-NEXT: s_lshr_b32 s87, s20, 16
-; VI-NEXT: s_lshr_b32 s7, s19, 16
-; VI-NEXT: s_lshr_b32 s9, s18, 16
-; VI-NEXT: s_lshr_b32 s11, s17, 16
-; VI-NEXT: s_lshr_b32 s13, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; VI-NEXT: s_lshr_b32 s34, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s71, s23, 16
+; VI-NEXT: s_lshr_b32 s82, s22, 16
+; VI-NEXT: s_lshr_b32 s84, s21, 16
+; VI-NEXT: s_lshr_b32 s85, s20, 16
+; VI-NEXT: s_lshr_b32 s86, s19, 16
+; VI-NEXT: s_lshr_b32 s87, s18, 16
+; VI-NEXT: s_lshr_b32 s7, s17, 16
+; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s63, s6, 16
-; VI-NEXT: s_lshr_b32 s60, s12, 16
+; VI-NEXT: s_lshr_b32 s11, s8, 16
+; VI-NEXT: s_lshr_b32 s13, s10, 16
+; VI-NEXT: s_lshr_b32 s15, s12, 16
; VI-NEXT: s_lshr_b32 s59, s14, 16
; VI-NEXT: s_lshr_b32 s58, s72, 16
; VI-NEXT: s_lshr_b32 s57, s74, 16
-; VI-NEXT: s_lshr_b32 s62, s76, 16
+; VI-NEXT: s_lshr_b32 s60, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s71, s69, 16
-; VI-NEXT: s_lshr_b32 s82, s80, 16
-; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s69, s35, 16
+; VI-NEXT: s_lshr_b32 s80, s70, 16
+; VI-NEXT: s_lshr_b32 s83, s81, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
-; VI-NEXT: v_writelane_b32 v33, s61, 1
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s60, 2
-; VI-NEXT: v_writelane_b32 v33, s59, 3
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_writelane_b32 v33, s59, 1
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s11, 16
+; VI-NEXT: s_lshl_b32 s41, s7, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s61, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s83
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s80
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s70
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
-; VI-NEXT: s_lshl_b32 s5, s62, 16
-; VI-NEXT: s_mov_b32 s77, s56
+; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_mov_b32 s73, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s75, s57
+; VI-NEXT: s_mov_b32 s78, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s72
; VI-NEXT: s_lshl_b32 s5, s58, 16
-; VI-NEXT: s_mov_b32 s78, s58
+; VI-NEXT: s_mov_b32 s77, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s14
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s59, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s12
-; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_mov_b32 s88, s60
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s10
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_mov_b32 s75, s61
; VI-NEXT: s_or_b32 s61, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, s15, 16
-; VI-NEXT: s_mov_b32 s88, s62
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s62, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s63, 16
-; VI-NEXT: s_mov_b32 s15, s63
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b32 s75, s61
+; VI-NEXT: s_mov_b32 s88, s60
+; VI-NEXT: s_mov_b32 s73, s56
+; VI-NEXT: s_mov_b32 s78, s57
+; VI-NEXT: s_mov_b32 s77, s58
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v27, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s13
-; VI-NEXT: v_mov_b32_e32 v2, s11
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_add_f16_sdwa v0, v0, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v27
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v27
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s9
+; VI-NEXT: v_mov_b32_e32 v2, s87
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v27
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_add_f16_sdwa v3, v3, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v27
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s87
+; VI-NEXT: v_mov_b32_e32 v4, s85
; VI-NEXT: v_add_f16_sdwa v4, v4, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v27
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s86
+; VI-NEXT: v_mov_b32_e32 v5, s84
; VI-NEXT: v_add_f16_sdwa v5, v5, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v27
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s84
+; VI-NEXT: v_mov_b32_e32 v6, s82
; VI-NEXT: v_add_f16_sdwa v6, v6, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v27
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s81
+; VI-NEXT: v_mov_b32_e32 v7, s71
; VI-NEXT: v_add_f16_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v27
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s70
+; VI-NEXT: v_mov_b32_e32 v8, s68
; VI-NEXT: v_add_f16_sdwa v8, v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v27
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s68
+; VI-NEXT: v_mov_b32_e32 v9, s34
; VI-NEXT: v_add_f16_sdwa v9, v9, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v27
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s31
+; VI-NEXT: v_mov_b32_e32 v10, vcc_hi
; VI-NEXT: v_add_f16_sdwa v10, v10, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v27
; VI-NEXT: v_or_b32_e32 v10, v11, v10
@@ -33033,31 +33465,31 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v11, v11, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v27
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s73
+; VI-NEXT: v_mov_b32_e32 v12, s75
; VI-NEXT: v_add_f16_sdwa v12, v12, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v27
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s77
+; VI-NEXT: v_mov_b32_e32 v13, s73
; VI-NEXT: v_add_f16_sdwa v13, v13, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v27
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s85
+; VI-NEXT: v_mov_b32_e32 v14, s83
; VI-NEXT: v_add_f16_sdwa v14, v14, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s83, v27
+; VI-NEXT: v_add_f16_e32 v15, s81, v27
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s82
+; VI-NEXT: v_mov_b32_e32 v15, s80
; VI-NEXT: v_add_f16_sdwa v15, v15, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s80, v27
+; VI-NEXT: v_add_f16_e32 v16, s70, v27
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s71
+; VI-NEXT: v_mov_b32_e32 v16, s69
; VI-NEXT: v_add_f16_sdwa v16, v16, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s69, v27
+; VI-NEXT: v_add_f16_e32 v17, s35, v27
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s35
+; VI-NEXT: v_mov_b32_e32 v17, s31
; VI-NEXT: v_add_f16_sdwa v17, v17, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s34, v27
+; VI-NEXT: v_add_f16_e32 v18, s30, v27
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s30
+; VI-NEXT: v_mov_b32_e32 v18, vcc_lo
; VI-NEXT: v_add_f16_sdwa v18, v18, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s91, v27
; VI-NEXT: v_or_b32_e32 v18, v19, v18
@@ -33069,49 +33501,38 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_sdwa v20, v20, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s76, v27
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s75
+; VI-NEXT: v_mov_b32_e32 v21, s78
; VI-NEXT: v_add_f16_sdwa v21, v21, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s74, v27
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s78
+; VI-NEXT: v_mov_b32_e32 v22, s77
; VI-NEXT: v_add_f16_sdwa v22, v22, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s72, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s14, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s12, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v24, v25, v24
-; VI-NEXT: v_mov_b32_e32 v25, s4
+; VI-NEXT: v_mov_b32_e32 v25, s15
; VI-NEXT: v_add_f16_sdwa v25, v25, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s10, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v25, v26, v25
-; VI-NEXT: v_mov_b32_e32 v26, s4
+; VI-NEXT: v_mov_b32_e32 v26, s13
; VI-NEXT: v_add_f16_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s8, v27
; VI-NEXT: v_or_b32_e32 v26, v28, v26
-; VI-NEXT: v_mov_b32_e32 v28, s15
+; VI-NEXT: v_mov_b32_e32 v28, s11
; VI-NEXT: v_add_f16_sdwa v28, v28, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s6, v27
; VI-NEXT: v_or_b32_e32 v27, v27, v28
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s88, s62
-; VI-NEXT: s_mov_b32 s77, s56
-; VI-NEXT: s_mov_b32 s75, s57
-; VI-NEXT: s_mov_b32 s78, s58
-; VI-NEXT: s_mov_b32 s15, s63
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -33144,7 +33565,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
+; VI-NEXT: .LBB47_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -33274,10 +33695,18 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v27, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v27 op_sel_hi:[1,0]
@@ -33307,10 +33736,8 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v25, s61, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v26, s62, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v27, s63, v27 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -33343,7 +33770,7 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -33434,11 +33861,16 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -33468,8 +33900,6 @@ define inreg <14 x i64> @bitcast_v56f16_to_v14i64_scalar(<56 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v26, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v27, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -34298,10 +34728,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_writelane_b32 v58, s49, 7
; SI-NEXT: v_writelane_b32 v58, s50, 8
; SI-NEXT: v_writelane_b32 v58, s51, 9
-; SI-NEXT: v_writelane_b32 v58, s52, 10
-; SI-NEXT: v_writelane_b32 v58, s53, 11
-; SI-NEXT: v_writelane_b32 v58, s30, 12
-; SI-NEXT: v_writelane_b32 v58, s31, 13
+; SI-NEXT: v_writelane_b32 v58, s30, 10
+; SI-NEXT: v_writelane_b32 v58, s31, 11
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: v_readfirstlane_b32 s41, v13
; SI-NEXT: v_readfirstlane_b32 s40, v12
@@ -34318,22 +34746,22 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s53, s41, 16
-; SI-NEXT: s_lshr_b32 s52, s15, 16
-; SI-NEXT: s_lshr_b32 s51, s13, 16
-; SI-NEXT: s_lshr_b32 s50, s11, 16
-; SI-NEXT: s_lshr_b32 s49, s9, 16
-; SI-NEXT: s_lshr_b32 s48, s7, 16
-; SI-NEXT: s_lshr_b32 s39, s5, 16
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s51, s41, 16
+; SI-NEXT: s_lshr_b32 s50, s15, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s39, s9, 16
+; SI-NEXT: s_lshr_b32 s38, s7, 16
+; SI-NEXT: s_lshr_b32 s37, s5, 16
+; SI-NEXT: s_lshr_b32 s36, s29, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s23, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[14:15], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
@@ -34348,8 +34776,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[14:15], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -34399,38 +34863,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -34459,27 +34893,27 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v22, s12
; SI-NEXT: v_mov_b32_e32 v24, s14
; SI-NEXT: v_mov_b32_e32 v26, s40
-; SI-NEXT: v_mov_b32_e32 v52, s53
-; SI-NEXT: v_mov_b32_e32 v53, s52
-; SI-NEXT: v_mov_b32_e32 v54, s51
-; SI-NEXT: v_mov_b32_e32 v55, s50
-; SI-NEXT: v_mov_b32_e32 v40, s49
-; SI-NEXT: v_mov_b32_e32 v41, s48
-; SI-NEXT: v_mov_b32_e32 v42, s39
+; SI-NEXT: v_mov_b32_e32 v52, s51
+; SI-NEXT: v_mov_b32_e32 v53, s50
+; SI-NEXT: v_mov_b32_e32 v54, s49
+; SI-NEXT: v_mov_b32_e32 v55, s48
+; SI-NEXT: v_mov_b32_e32 v40, s39
+; SI-NEXT: v_mov_b32_e32 v41, s38
+; SI-NEXT: v_mov_b32_e32 v42, s37
; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mov_b32_e32 v43, s38
+; SI-NEXT: v_mov_b32_e32 v43, s36
; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_mov_b32_e32 v44, s37
+; SI-NEXT: v_mov_b32_e32 v44, s35
; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v45, s36
+; SI-NEXT: v_mov_b32_e32 v45, s34
; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v46, s35
+; SI-NEXT: v_mov_b32_e32 v46, s31
; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v47, s34
+; SI-NEXT: v_mov_b32_e32 v47, s30
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v56, s31
+; SI-NEXT: v_mov_b32_e32 v56, vcc_hi
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v57, s30
+; SI-NEXT: v_mov_b32_e32 v57, vcc_lo
; SI-NEXT: v_mov_b32_e32 v50, s92
; SI-NEXT: v_mov_b32_e32 v49, s90
; SI-NEXT: v_mov_b32_e32 v48, s88
@@ -34494,7 +34928,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v30, s46
; SI-NEXT: v_mov_b32_e32 v29, s44
; SI-NEXT: v_mov_b32_e32 v28, s42
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -34577,7 +35011,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v52
-; SI-NEXT: v_readlane_b32 s30, v58, 12
+; SI-NEXT: v_readlane_b32 s30, v58, 10
; SI-NEXT: v_or_b32_e32 v5, v5, v39
; SI-NEXT: v_or_b32_e32 v7, v7, v38
; SI-NEXT: v_or_b32_e32 v9, v9, v37
@@ -34590,9 +35024,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; SI-NEXT: v_or_b32_e32 v23, v23, v30
; SI-NEXT: v_or_b32_e32 v25, v25, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: v_readlane_b32 s31, v58, 13
-; SI-NEXT: v_readlane_b32 s53, v58, 11
-; SI-NEXT: v_readlane_b32 s52, v58, 10
+; SI-NEXT: v_readlane_b32 s31, v58, 11
; SI-NEXT: v_readlane_b32 s51, v58, 9
; SI-NEXT: v_readlane_b32 s50, v58, 8
; SI-NEXT: v_readlane_b32 s49, v58, 7
@@ -34623,10 +35055,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
-; VI-NEXT: v_writelane_b32 v56, s34, 0
-; VI-NEXT: v_writelane_b32 v56, s35, 1
-; VI-NEXT: v_writelane_b32 v56, s30, 2
-; VI-NEXT: v_writelane_b32 v56, s31, 3
+; VI-NEXT: v_writelane_b32 v56, s30, 0
+; VI-NEXT: v_writelane_b32 v56, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s9, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
@@ -34643,7 +35073,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s9, 16
; VI-NEXT: s_lshr_b32 s74, s8, 16
@@ -34666,15 +35096,51 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s61, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
; VI-NEXT: s_lshr_b32 s62, s23, 16
-; VI-NEXT: s_lshr_b32 s30, s22, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s22, 16
; VI-NEXT: s_lshr_b32 s63, s21, 16
-; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s20, 16
; VI-NEXT: s_lshr_b32 s72, s19, 16
-; VI-NEXT: s_lshr_b32 s34, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s18, 16
; VI-NEXT: s_lshr_b32 s73, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[42:43], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[42:43], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; VI-NEXT: s_cselect_b32 s42, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s42, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[26:27], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -34717,38 +35183,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v42, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v44, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -34777,10 +35213,10 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v25, s11
; VI-NEXT: v_mov_b32_e32 v26, s8
; VI-NEXT: v_mov_b32_e32 v27, s9
-; VI-NEXT: v_mov_b32_e32 v44, s35
-; VI-NEXT: v_mov_b32_e32 v42, s34
-; VI-NEXT: v_mov_b32_e32 v40, s31
-; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: v_mov_b32_e32 v44, s31
+; VI-NEXT: v_mov_b32_e32 v42, s30
+; VI-NEXT: v_mov_b32_e32 v40, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v54, vcc_lo
; VI-NEXT: v_mov_b32_e32 v52, s91
; VI-NEXT: v_mov_b32_e32 v50, s90
; VI-NEXT: v_mov_b32_e32 v48, s89
@@ -34805,7 +35241,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v35, s46
; VI-NEXT: v_mov_b32_e32 v33, s45
; VI-NEXT: v_mov_b32_e32 v31, s44
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
; VI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
@@ -34859,7 +35295,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v33
; VI-NEXT: v_or_b32_sdwa v26, v26, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v28, 16, v31
-; VI-NEXT: v_readlane_b32 s30, v56, 2
+; VI-NEXT: v_readlane_b32 s30, v56, 0
; VI-NEXT: v_or_b32_sdwa v7, v7, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v11, v11, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -34871,9 +35307,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; VI-NEXT: v_or_b32_sdwa v23, v23, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v25, v25, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v56, 3
-; VI-NEXT: v_readlane_b32 s35, v56, 1
-; VI-NEXT: v_readlane_b32 s34, v56, 0
+; VI-NEXT: v_readlane_b32 s31, v56, 1
; VI-NEXT: s_or_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -34907,7 +35341,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s9, 16
; GFX9-NEXT: s_lshr_b32 s74, s8, 16
@@ -34937,8 +35371,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s94, s18, 16
; GFX9-NEXT: s_lshr_b32 s73, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[42:43], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[42:43], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; GFX9-NEXT: s_cselect_b32 s42, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s42, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[26:27], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -34981,38 +35451,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v44, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -35069,7 +35509,7 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v35, s46
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -35152,12 +35592,12 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s11, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s11, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s10, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s9, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s8, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s63, s6, 16
@@ -35183,9 +35623,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s89, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
@@ -35228,38 +35703,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -35286,9 +35731,9 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s73 :: v_dual_mov_b32 v39, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s72 :: v_dual_mov_b32 v37, s42
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s63 :: v_dual_mov_b32 v35, s41
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s61 :: v_dual_mov_b32 v33, s40
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v70, s59 :: v_dual_mov_b32 v31, s15
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s61 :: v_dual_mov_b32 v33, s15
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v70, s59 :: v_dual_mov_b32 v31, s14
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v68.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v71.l
@@ -35336,12 +35781,12 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s11, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s11, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s10, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s9, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s8, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s63, s6, 16
@@ -35367,9 +35812,44 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s89, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[10:11], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[8:9], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[6:7], 1.0
@@ -35412,38 +35892,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v31
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v0
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v31, s2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v10, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s18 :: v_dual_mov_b32 v8, s22
@@ -35471,8 +35921,8 @@ define inreg <56 x i16> @bitcast_v14f64_to_v56i16_scalar(<14 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v65, s56 :: v_dual_mov_b32 v52, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s44 :: v_dual_mov_b32 v48, s42
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, s43 :: v_dual_mov_b32 v38, s41
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s40 :: v_dual_mov_b32 v35, s15
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s15 :: v_dual_mov_b32 v35, s14
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v31, 0xffff, v31
; GFX11-FAKE16-NEXT: v_and_b32_e32 v29, 0xffff, v29
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -36828,10 +37278,8 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v28, s83, 25
; SI-NEXT: v_writelane_b32 v28, s84, 26
; SI-NEXT: v_writelane_b32 v28, s85, 27
-; SI-NEXT: v_writelane_b32 v28, s86, 28
-; SI-NEXT: v_writelane_b32 v28, s87, 29
-; SI-NEXT: v_writelane_b32 v28, s30, 30
-; SI-NEXT: v_writelane_b32 v28, s31, 31
+; SI-NEXT: v_writelane_b32 v28, s30, 28
+; SI-NEXT: v_writelane_b32 v28, s31, 29
; SI-NEXT: v_readfirstlane_b32 s7, v13
; SI-NEXT: v_readfirstlane_b32 s9, v12
; SI-NEXT: v_readfirstlane_b32 s11, v11
@@ -36843,23 +37291,23 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s88, v5
; SI-NEXT: v_readfirstlane_b32 s91, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s31, v2
-; SI-NEXT: v_readfirstlane_b32 s69, v1
-; SI-NEXT: v_readfirstlane_b32 s80, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_hi, v2
+; SI-NEXT: v_readfirstlane_b32 s35, v1
+; SI-NEXT: v_readfirstlane_b32 s70, v0
; SI-NEXT: s_lshr_b32 s78, s29, 16
; SI-NEXT: s_lshr_b32 s89, s28, 16
; SI-NEXT: s_lshr_b32 s92, s27, 16
; SI-NEXT: s_lshr_b32 s95, s26, 16
-; SI-NEXT: s_lshr_b32 s34, s25, 16
-; SI-NEXT: s_lshr_b32 s68, s24, 16
-; SI-NEXT: s_lshr_b32 s70, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 s30, s25, 16
+; SI-NEXT: s_lshr_b32 s34, s24, 16
+; SI-NEXT: s_lshr_b32 s68, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -36871,42 +37319,42 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s79, s88, 16
; SI-NEXT: s_lshr_b32 s90, s91, 16
; SI-NEXT: s_lshr_b32 s93, s94, 16
-; SI-NEXT: s_lshr_b32 s30, s31, 16
-; SI-NEXT: s_lshr_b32 s35, s69, 16
-; SI-NEXT: s_lshr_b32 s71, s80, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, vcc_hi, 16
+; SI-NEXT: s_lshr_b32 s31, s35, 16
+; SI-NEXT: s_lshr_b32 s69, s70, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -36920,14 +37368,14 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -36962,56 +37410,65 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -37033,20 +37490,20 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s80, s80, 3
+; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s80, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_and_b32 s4, s70, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s69, s69, 3
+; SI-NEXT: s_add_i32 s35, s35, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_and_b32 s4, s35, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s31, s31, 3
+; SI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s31, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s94, s94, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
@@ -37104,8 +37561,8 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s63, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v28, 30
+; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v28, 28
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -37134,9 +37591,7 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v25, s61
; SI-NEXT: v_mov_b32_e32 v26, s62
; SI-NEXT: v_mov_b32_e32 v27, s63
-; SI-NEXT: v_readlane_b32 s31, v28, 31
-; SI-NEXT: v_readlane_b32 s87, v28, 29
-; SI-NEXT: v_readlane_b32 s86, v28, 28
+; SI-NEXT: v_readlane_b32 s31, v28, 29
; SI-NEXT: v_readlane_b32 s85, v28, 27
; SI-NEXT: v_readlane_b32 s84, v28, 26
; SI-NEXT: v_readlane_b32 s83, v28, 25
@@ -37170,9 +37625,6 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v56i16_to_v14f64_scalar:
; VI: ; %bb.0:
@@ -37213,54 +37665,54 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v28, s87, 29
; VI-NEXT: v_writelane_b32 v28, s30, 30
; VI-NEXT: v_writelane_b32 v28, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s86, v13
-; VI-NEXT: v_readfirstlane_b32 s6, v12
+; VI-NEXT: v_readfirstlane_b32 s81, v13
+; VI-NEXT: v_readfirstlane_b32 s83, v12
; VI-NEXT: v_readfirstlane_b32 s9, v11
-; VI-NEXT: v_readfirstlane_b32 s8, v10
-; VI-NEXT: v_readfirstlane_b32 s87, v9
-; VI-NEXT: v_readfirstlane_b32 s12, v8
+; VI-NEXT: v_readfirstlane_b32 s84, v10
+; VI-NEXT: v_readfirstlane_b32 s85, v9
+; VI-NEXT: v_readfirstlane_b32 s8, v8
; VI-NEXT: v_readfirstlane_b32 s73, v7
-; VI-NEXT: v_readfirstlane_b32 s77, v6
-; VI-NEXT: v_readfirstlane_b32 s88, v5
+; VI-NEXT: v_readfirstlane_b32 s75, v6
+; VI-NEXT: v_readfirstlane_b32 s77, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s81, v1
-; VI-NEXT: v_readfirstlane_b32 s84, v0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s71, v1
+; VI-NEXT: v_readfirstlane_b32 s82, v0
; VI-NEXT: s_lshr_b32 s79, s29, 16
; VI-NEXT: s_lshr_b32 s90, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s67, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s80, s24, 16
-; VI-NEXT: s_lshr_b32 s82, s23, 16
-; VI-NEXT: s_lshr_b32 s85, s22, 16
-; VI-NEXT: s_lshr_b32 s7, s21, 16
-; VI-NEXT: s_lshr_b32 s11, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s66, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s69, s23, 16
+; VI-NEXT: s_lshr_b32 s80, s22, 16
+; VI-NEXT: s_lshr_b32 s86, s21, 16
+; VI-NEXT: s_lshr_b32 s87, s20, 16
; VI-NEXT: s_lshr_b32 s13, s19, 16
; VI-NEXT: s_lshr_b32 s15, s18, 16
-; VI-NEXT: s_lshr_b32 s10, s17, 16
-; VI-NEXT: s_lshr_b32 s75, s16, 16
-; VI-NEXT: s_lshr_b32 s64, s86, 16
-; VI-NEXT: s_lshr_b32 s65, s6, 16
-; VI-NEXT: s_lshr_b32 s66, s9, 16
-; VI-NEXT: s_lshr_b32 s14, s8, 16
-; VI-NEXT: s_lshr_b32 s72, s87, 16
-; VI-NEXT: s_lshr_b32 s74, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s17, 16
+; VI-NEXT: s_lshr_b32 s7, s16, 16
+; VI-NEXT: s_lshr_b32 s64, s81, 16
+; VI-NEXT: s_lshr_b32 s10, s83, 16
+; VI-NEXT: s_lshr_b32 s12, s9, 16
+; VI-NEXT: s_lshr_b32 s14, s84, 16
+; VI-NEXT: s_lshr_b32 s72, s85, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
; VI-NEXT: s_lshr_b32 s76, s73, 16
-; VI-NEXT: s_lshr_b32 s78, s77, 16
-; VI-NEXT: s_lshr_b32 s89, s88, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s68, s69, 16
-; VI-NEXT: s_lshr_b32 s71, s81, 16
-; VI-NEXT: s_lshr_b32 s83, s84, 16
+; VI-NEXT: s_lshr_b32 s78, s75, 16
+; VI-NEXT: s_lshr_b32 s89, s77, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s65, s35, 16
+; VI-NEXT: s_lshr_b32 s67, s71, 16
+; VI-NEXT: s_lshr_b32 s70, s82, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
; VI-NEXT: s_lshl_b32 s5, s15, 16
@@ -37268,32 +37720,32 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s19
; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s10, 16
+; VI-NEXT: s_lshl_b32 s41, s88, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
; VI-NEXT: s_lshl_b32 s5, s90, 16
@@ -37301,58 +37753,146 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s84
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s82
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s81
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s71
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s88
+; VI-NEXT: s_and_b32 s4, 0xffff, s77
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s77
+; VI-NEXT: s_and_b32 s4, 0xffff, s75
; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s73
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s57, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_and_b32 s4, 0xffff, s85
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_and_b32 s4, 0xffff, s84
; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: v_writelane_b32 v29, s26, 0
+; VI-NEXT: s_mov_b32 s26, s27
+; VI-NEXT: s_mov_b32 s27, s77
+; VI-NEXT: v_writelane_b32 v29, s91, 1
+; VI-NEXT: s_mov_b32 s91, s88
+; VI-NEXT: s_mov_b32 s88, s28
+; VI-NEXT: s_mov_b32 s28, s75
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s81
+; VI-NEXT: s_mov_b32 s81, s68
+; VI-NEXT: s_mov_b32 s68, s34
+; VI-NEXT: s_mov_b32 s34, s31
+; VI-NEXT: s_mov_b32 s31, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s90
+; VI-NEXT: s_mov_b32 s90, s89
+; VI-NEXT: s_mov_b32 s89, s79
+; VI-NEXT: s_mov_b32 s79, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s72
+; VI-NEXT: s_mov_b32 s72, s14
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s10, s64
+; VI-NEXT: s_mov_b32 s9, s83
+; VI-NEXT: s_mov_b32 s83, s69
+; VI-NEXT: s_mov_b32 s69, s65
+; VI-NEXT: s_mov_b32 s6, s13
+; VI-NEXT: s_mov_b32 s13, s84
+; VI-NEXT: s_mov_b32 s84, s70
+; VI-NEXT: s_mov_b32 s70, s66
+; VI-NEXT: s_mov_b32 s75, s73
+; VI-NEXT: s_mov_b32 s73, s8
+; VI-NEXT: s_mov_b32 s8, s15
+; VI-NEXT: s_mov_b32 s15, s85
+; VI-NEXT: s_mov_b32 s85, s80
+; VI-NEXT: s_mov_b32 s80, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s80
+; VI-NEXT: s_mov_b32 s80, s85
+; VI-NEXT: s_mov_b32 s85, s15
+; VI-NEXT: s_mov_b32 s15, s8
+; VI-NEXT: s_mov_b32 s8, s73
+; VI-NEXT: s_mov_b32 s73, s75
+; VI-NEXT: s_mov_b32 s66, s70
+; VI-NEXT: s_mov_b32 s70, s84
+; VI-NEXT: s_mov_b32 s84, s13
+; VI-NEXT: s_mov_b32 s13, s6
+; VI-NEXT: s_mov_b32 s65, s69
+; VI-NEXT: s_mov_b32 s69, s83
+; VI-NEXT: s_mov_b32 s83, s9
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s64, s10
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s14, s72
+; VI-NEXT: s_mov_b32 s72, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s79
+; VI-NEXT: s_mov_b32 s79, s89
+; VI-NEXT: s_mov_b32 s89, s90
+; VI-NEXT: s_mov_b32 s90, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s31
+; VI-NEXT: s_mov_b32 s31, s34
+; VI-NEXT: s_mov_b32 s34, s68
+; VI-NEXT: s_mov_b32 s68, s81
+; VI-NEXT: s_mov_b32 s81, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s75, s28
+; VI-NEXT: s_mov_b32 s28, s88
+; VI-NEXT: s_mov_b32 s88, s91
+; VI-NEXT: v_readlane_b32 s91, v29, 1
+; VI-NEXT: s_mov_b32 s77, s27
+; VI-NEXT: s_mov_b32 s27, s26
+; VI-NEXT: v_readlane_b32 s26, v29, 0
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s10, s10, 16
-; VI-NEXT: s_or_b32 s5, s10, s5
+; VI-NEXT: s_lshl_b32 s6, s88, 16
+; VI-NEXT: s_or_b32 s5, s6, s5
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s36, s4, 0x30000
; VI-NEXT: s_add_i32 s37, s5, 0x30000
@@ -37367,42 +37907,42 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -37414,37 +37954,37 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s29, 0xffff
; VI-NEXT: s_lshl_b32 s5, s79, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s84, s84, 3
+; VI-NEXT: s_add_i32 s82, s82, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s84, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, s82, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s81, s81, 3
+; VI-NEXT: s_add_i32 s71, s71, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s81, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, s71, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s69, s69, 3
+; VI-NEXT: s_add_i32 s35, s35, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s69, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_and_b32 s4, s35, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s65, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s34, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s30, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s91, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s88, s88, 3
+; VI-NEXT: s_add_i32 s88, s77, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s88, 0xffff
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s77, s77, 3
+; VI-NEXT: s_add_i32 s77, s75, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
; VI-NEXT: s_lshl_b32 s5, s78, 16
@@ -37454,17 +37994,17 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_and_b32 s4, s75, 0xffff
; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s12, 3
+; VI-NEXT: s_add_i32 s73, s8, 3
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s87, 3
+; VI-NEXT: s_add_i32 s15, s85, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
; VI-NEXT: s_lshl_b32 s5, s72, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s8, 3
+; VI-NEXT: s_add_i32 s13, s84, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
; VI-NEXT: s_lshl_b32 s5, s14, 16
@@ -37472,20 +38012,20 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_add_i32 s11, s9, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s66, 16
+; VI-NEXT: s_lshl_b32 s5, s12, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s6, 3
+; VI-NEXT: s_add_i32 s9, s83, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_lshl_b32 s5, s10, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s86, 3
+; VI-NEXT: s_add_i32 s7, s81, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s64, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s63, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v28, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -37552,96 +38092,6 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
-; VI-NEXT: s_mov_b32 vcc_lo, s64
-; VI-NEXT: v_writelane_b32 v29, s75, 0
-; VI-NEXT: s_mov_b32 vcc_hi, s65
-; VI-NEXT: s_mov_b32 s75, s73
-; VI-NEXT: s_mov_b32 s73, s12
-; VI-NEXT: v_writelane_b32 v29, s25, 1
-; VI-NEXT: s_mov_b32 s25, s91
-; VI-NEXT: s_mov_b32 s91, s26
-; VI-NEXT: s_mov_b32 s26, s27
-; VI-NEXT: s_mov_b32 s27, s88
-; VI-NEXT: s_mov_b32 s88, s28
-; VI-NEXT: s_mov_b32 s28, s77
-; VI-NEXT: s_mov_b32 s77, s29
-; VI-NEXT: s_mov_b32 s29, s15
-; VI-NEXT: s_mov_b32 s15, s87
-; VI-NEXT: s_mov_b32 s87, s85
-; VI-NEXT: s_mov_b32 s85, s82
-; VI-NEXT: s_mov_b32 s82, s71
-; VI-NEXT: s_mov_b32 s71, s68
-; VI-NEXT: s_mov_b32 s68, s35
-; VI-NEXT: s_mov_b32 s35, s31
-; VI-NEXT: s_mov_b32 s31, s30
-; VI-NEXT: s_mov_b32 s30, s90
-; VI-NEXT: s_mov_b32 s90, s89
-; VI-NEXT: s_mov_b32 s89, s79
-; VI-NEXT: s_mov_b32 s79, s78
-; VI-NEXT: s_mov_b32 s78, s76
-; VI-NEXT: s_mov_b32 s76, s74
-; VI-NEXT: s_mov_b32 s74, s72
-; VI-NEXT: s_mov_b32 s72, s14
-; VI-NEXT: s_mov_b32 s14, s66
-; VI-NEXT: s_mov_b32 s12, s10
-; VI-NEXT: s_mov_b32 s10, s13
-; VI-NEXT: s_mov_b32 s13, s8
-; VI-NEXT: s_mov_b32 s8, s11
-; VI-NEXT: s_mov_b32 s11, s9
-; VI-NEXT: s_mov_b32 s9, s6
-; VI-NEXT: s_mov_b32 s6, s7
-; VI-NEXT: s_mov_b32 s7, s86
-; VI-NEXT: s_mov_b32 s86, s83
-; VI-NEXT: s_mov_b32 s83, s80
-; VI-NEXT: s_mov_b32 s80, s70
-; VI-NEXT: s_mov_b32 s70, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, s70
-; VI-NEXT: s_mov_b32 s70, s80
-; VI-NEXT: s_mov_b32 s80, s83
-; VI-NEXT: s_mov_b32 s83, s86
-; VI-NEXT: s_mov_b32 s86, s7
-; VI-NEXT: s_mov_b32 s7, s6
-; VI-NEXT: s_mov_b32 s6, s9
-; VI-NEXT: s_mov_b32 s9, s11
-; VI-NEXT: s_mov_b32 s11, s8
-; VI-NEXT: s_mov_b32 s8, s13
-; VI-NEXT: s_mov_b32 s13, s10
-; VI-NEXT: s_mov_b32 s10, s12
-; VI-NEXT: s_mov_b32 s66, s14
-; VI-NEXT: s_mov_b32 s14, s72
-; VI-NEXT: s_mov_b32 s72, s74
-; VI-NEXT: s_mov_b32 s74, s76
-; VI-NEXT: s_mov_b32 s76, s78
-; VI-NEXT: s_mov_b32 s78, s79
-; VI-NEXT: s_mov_b32 s79, s89
-; VI-NEXT: s_mov_b32 s89, s90
-; VI-NEXT: s_mov_b32 s90, s30
-; VI-NEXT: s_mov_b32 s30, s31
-; VI-NEXT: s_mov_b32 s31, s35
-; VI-NEXT: s_mov_b32 s35, s68
-; VI-NEXT: s_mov_b32 s68, s71
-; VI-NEXT: s_mov_b32 s71, s82
-; VI-NEXT: s_mov_b32 s82, s85
-; VI-NEXT: s_mov_b32 s85, s87
-; VI-NEXT: s_mov_b32 s87, s15
-; VI-NEXT: s_mov_b32 s15, s29
-; VI-NEXT: s_mov_b32 s29, s77
-; VI-NEXT: s_mov_b32 s77, s28
-; VI-NEXT: s_mov_b32 s28, s88
-; VI-NEXT: s_mov_b32 s88, s27
-; VI-NEXT: s_mov_b32 s27, s26
-; VI-NEXT: s_mov_b32 s26, s91
-; VI-NEXT: s_mov_b32 s91, s25
-; VI-NEXT: v_readlane_b32 s25, v29, 1
-; VI-NEXT: s_mov_b32 s12, s73
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: v_readlane_b32 s75, v29, 0
-; VI-NEXT: s_mov_b32 s65, vcc_hi
-; VI-NEXT: s_mov_b32 s64, vcc_lo
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v56i16_to_v14f64_scalar:
; GFX9: ; %bb.0:
@@ -37733,10 +38183,18 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -37765,10 +38223,8 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v25, s61, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v26, s62, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -37801,7 +38257,7 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -37892,11 +38348,16 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -37926,8 +38387,6 @@ define inreg <14 x double> @bitcast_v56i16_to_v14f64_scalar(<56 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v26, s26, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v27, s27, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -38756,10 +39215,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_writelane_b32 v58, s49, 7
; SI-NEXT: v_writelane_b32 v58, s50, 8
; SI-NEXT: v_writelane_b32 v58, s51, 9
-; SI-NEXT: v_writelane_b32 v58, s52, 10
-; SI-NEXT: v_writelane_b32 v58, s53, 11
-; SI-NEXT: v_writelane_b32 v58, s30, 12
-; SI-NEXT: v_writelane_b32 v58, s31, 13
+; SI-NEXT: v_writelane_b32 v58, s30, 10
+; SI-NEXT: v_writelane_b32 v58, s31, 11
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: v_readfirstlane_b32 s41, v13
; SI-NEXT: v_readfirstlane_b32 s40, v12
@@ -38776,22 +39233,22 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s5, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s4, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_lshr_b32 s53, s41, 16
-; SI-NEXT: s_lshr_b32 s52, s15, 16
-; SI-NEXT: s_lshr_b32 s51, s13, 16
-; SI-NEXT: s_lshr_b32 s50, s11, 16
-; SI-NEXT: s_lshr_b32 s49, s9, 16
-; SI-NEXT: s_lshr_b32 s48, s7, 16
-; SI-NEXT: s_lshr_b32 s39, s5, 16
-; SI-NEXT: s_lshr_b32 s38, s29, 16
-; SI-NEXT: s_lshr_b32 s37, s27, 16
-; SI-NEXT: s_lshr_b32 s36, s25, 16
-; SI-NEXT: s_lshr_b32 s35, s23, 16
-; SI-NEXT: s_lshr_b32 s34, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s19, 16
-; SI-NEXT: s_lshr_b32 s30, s17, 16
+; SI-NEXT: s_lshr_b32 s51, s41, 16
+; SI-NEXT: s_lshr_b32 s50, s15, 16
+; SI-NEXT: s_lshr_b32 s49, s13, 16
+; SI-NEXT: s_lshr_b32 s48, s11, 16
+; SI-NEXT: s_lshr_b32 s39, s9, 16
+; SI-NEXT: s_lshr_b32 s38, s7, 16
+; SI-NEXT: s_lshr_b32 s37, s5, 16
+; SI-NEXT: s_lshr_b32 s36, s29, 16
+; SI-NEXT: s_lshr_b32 s35, s27, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s31, s23, 16
+; SI-NEXT: s_lshr_b32 s30, s21, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s19, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s17, 16
; SI-NEXT: s_lshr_b64 s[42:43], s[40:41], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[14:15], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[12:13], 16
@@ -38806,8 +39263,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[88:89], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[90:91], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[94:95], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[94:95], -1
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $vcc_hi
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr31
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[94:95], s[94:95], exec
+; SI-NEXT: s_cselect_b32 s43, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s43, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[14:15], 1.0
; SI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -38857,38 +39350,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr31
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -38917,27 +39380,27 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v22, s12
; SI-NEXT: v_mov_b32_e32 v24, s14
; SI-NEXT: v_mov_b32_e32 v26, s40
-; SI-NEXT: v_mov_b32_e32 v52, s53
-; SI-NEXT: v_mov_b32_e32 v53, s52
-; SI-NEXT: v_mov_b32_e32 v54, s51
-; SI-NEXT: v_mov_b32_e32 v55, s50
-; SI-NEXT: v_mov_b32_e32 v40, s49
-; SI-NEXT: v_mov_b32_e32 v41, s48
-; SI-NEXT: v_mov_b32_e32 v42, s39
+; SI-NEXT: v_mov_b32_e32 v52, s51
+; SI-NEXT: v_mov_b32_e32 v53, s50
+; SI-NEXT: v_mov_b32_e32 v54, s49
+; SI-NEXT: v_mov_b32_e32 v55, s48
+; SI-NEXT: v_mov_b32_e32 v40, s39
+; SI-NEXT: v_mov_b32_e32 v41, s38
+; SI-NEXT: v_mov_b32_e32 v42, s37
; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mov_b32_e32 v43, s38
+; SI-NEXT: v_mov_b32_e32 v43, s36
; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_mov_b32_e32 v44, s37
+; SI-NEXT: v_mov_b32_e32 v44, s35
; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mov_b32_e32 v45, s36
+; SI-NEXT: v_mov_b32_e32 v45, s34
; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v46, s35
+; SI-NEXT: v_mov_b32_e32 v46, s31
; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v47, s34
+; SI-NEXT: v_mov_b32_e32 v47, s30
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v56, s31
+; SI-NEXT: v_mov_b32_e32 v56, vcc_hi
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v57, s30
+; SI-NEXT: v_mov_b32_e32 v57, vcc_lo
; SI-NEXT: v_mov_b32_e32 v50, s92
; SI-NEXT: v_mov_b32_e32 v49, s90
; SI-NEXT: v_mov_b32_e32 v48, s88
@@ -38952,7 +39415,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v30, s46
; SI-NEXT: v_mov_b32_e32 v29, s44
; SI-NEXT: v_mov_b32_e32 v28, s42
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -39035,7 +39498,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v27
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v52
-; SI-NEXT: v_readlane_b32 s30, v58, 12
+; SI-NEXT: v_readlane_b32 s30, v58, 10
; SI-NEXT: v_or_b32_e32 v5, v5, v39
; SI-NEXT: v_or_b32_e32 v7, v7, v38
; SI-NEXT: v_or_b32_e32 v9, v9, v37
@@ -39048,9 +39511,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; SI-NEXT: v_or_b32_e32 v23, v23, v30
; SI-NEXT: v_or_b32_e32 v25, v25, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v28
-; SI-NEXT: v_readlane_b32 s31, v58, 13
-; SI-NEXT: v_readlane_b32 s53, v58, 11
-; SI-NEXT: v_readlane_b32 s52, v58, 10
+; SI-NEXT: v_readlane_b32 s31, v58, 11
; SI-NEXT: v_readlane_b32 s51, v58, 9
; SI-NEXT: v_readlane_b32 s50, v58, 8
; SI-NEXT: v_readlane_b32 s49, v58, 7
@@ -39081,10 +39542,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
-; VI-NEXT: v_writelane_b32 v56, s34, 0
-; VI-NEXT: v_writelane_b32 v56, s35, 1
-; VI-NEXT: v_writelane_b32 v56, s30, 2
-; VI-NEXT: v_writelane_b32 v56, s31, 3
+; VI-NEXT: v_writelane_b32 v56, s30, 0
+; VI-NEXT: v_writelane_b32 v56, s31, 1
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: v_readfirstlane_b32 s9, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
@@ -39101,7 +39560,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s44, s9, 16
; VI-NEXT: s_lshr_b32 s74, s8, 16
@@ -39124,15 +39583,51 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: s_lshr_b32 s61, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
; VI-NEXT: s_lshr_b32 s62, s23, 16
-; VI-NEXT: s_lshr_b32 s30, s22, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s22, 16
; VI-NEXT: s_lshr_b32 s63, s21, 16
-; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s20, 16
; VI-NEXT: s_lshr_b32 s72, s19, 16
-; VI-NEXT: s_lshr_b32 s34, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s18, 16
; VI-NEXT: s_lshr_b32 s73, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 s31, s16, 16
+; VI-NEXT: s_mov_b64 s[42:43], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[42:43], -1
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr45
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr44
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; VI-NEXT: s_cselect_b32 s42, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s42, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[26:27], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -39175,38 +39670,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v42, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v47, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v44, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr45
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr44
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -39235,10 +39700,10 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v25, s11
; VI-NEXT: v_mov_b32_e32 v26, s8
; VI-NEXT: v_mov_b32_e32 v27, s9
-; VI-NEXT: v_mov_b32_e32 v44, s35
-; VI-NEXT: v_mov_b32_e32 v42, s34
-; VI-NEXT: v_mov_b32_e32 v40, s31
-; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: v_mov_b32_e32 v44, s31
+; VI-NEXT: v_mov_b32_e32 v42, s30
+; VI-NEXT: v_mov_b32_e32 v40, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v54, vcc_lo
; VI-NEXT: v_mov_b32_e32 v52, s91
; VI-NEXT: v_mov_b32_e32 v50, s90
; VI-NEXT: v_mov_b32_e32 v48, s89
@@ -39263,7 +39728,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v35, s46
; VI-NEXT: v_mov_b32_e32 v33, s45
; VI-NEXT: v_mov_b32_e32 v31, s44
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
; VI-NEXT: v_lshlrev_b32_e32 v42, 16, v42
; VI-NEXT: v_lshlrev_b32_e32 v40, 16, v40
@@ -39317,7 +39782,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v33
; VI-NEXT: v_or_b32_sdwa v26, v26, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v28, 16, v31
-; VI-NEXT: v_readlane_b32 s30, v56, 2
+; VI-NEXT: v_readlane_b32 s30, v56, 0
; VI-NEXT: v_or_b32_sdwa v7, v7, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v11, v11, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -39329,9 +39794,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; VI-NEXT: v_or_b32_sdwa v23, v23, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v25, v25, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v56, 3
-; VI-NEXT: v_readlane_b32 s35, v56, 1
-; VI-NEXT: v_readlane_b32 s34, v56, 0
+; VI-NEXT: v_readlane_b32 s31, v56, 1
; VI-NEXT: s_or_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -39365,7 +39828,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s44, s9, 16
; GFX9-NEXT: s_lshr_b32 s74, s8, 16
@@ -39395,8 +39858,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s94, s18, 16
; GFX9-NEXT: s_lshr_b32 s73, s17, 16
; GFX9-NEXT: s_lshr_b32 s95, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[42:43], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[42:43], -1
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr45
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr44
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[42:43], s[42:43], exec
+; GFX9-NEXT: s_cselect_b32 s42, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s42, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[26:27], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[22:23], s[12:13], 1.0
@@ -39439,38 +39938,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v42, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v44, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr45
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr44
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -39527,7 +39996,7 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v35, s46
; GFX9-NEXT: v_mov_b32_e32 v33, s45
; GFX9-NEXT: v_mov_b32_e32 v31, s44
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -39610,12 +40079,12 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s11, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s11, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s61, s10, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s9, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s62, s8, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s63, s6, 16
@@ -39641,9 +40110,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s89, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s40, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[22:23], s[6:7], 1.0
@@ -39686,38 +40190,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -39744,9 +40218,9 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s73 :: v_dual_mov_b32 v39, s43
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s72 :: v_dual_mov_b32 v37, s42
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s63 :: v_dual_mov_b32 v35, s41
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s61 :: v_dual_mov_b32 v33, s40
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v70, s59 :: v_dual_mov_b32 v31, s15
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s61 :: v_dual_mov_b32 v33, s15
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v70, s59 :: v_dual_mov_b32 v31, s14
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v68.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v71.l
@@ -39794,12 +40268,12 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s14, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s11, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s11, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s61, s10, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s9, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s15, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s62, s8, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s63, s6, 16
@@ -39825,9 +40299,44 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s89, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s60, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s90, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s40, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[23:24], s[10:11], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[8:9], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[6:7], 1.0
@@ -39870,38 +40379,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v31
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v0
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v31, s2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v10, s20
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s18 :: v_dual_mov_b32 v8, s22
@@ -39929,8 +40408,8 @@ define inreg <56 x half> @bitcast_v14f64_to_v56f16_scalar(<14 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v65, s56 :: v_dual_mov_b32 v52, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s44 :: v_dual_mov_b32 v48, s42
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, s43 :: v_dual_mov_b32 v38, s41
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s40 :: v_dual_mov_b32 v35, s15
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, s15 :: v_dual_mov_b32 v35, s14
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v31, 0xffff, v31
; GFX11-FAKE16-NEXT: v_and_b32_e32 v29, 0xffff, v29
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -41423,10 +41902,8 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_writelane_b32 v32, s83, 25
; SI-NEXT: v_writelane_b32 v32, s84, 26
; SI-NEXT: v_writelane_b32 v32, s85, 27
-; SI-NEXT: v_writelane_b32 v32, s86, 28
-; SI-NEXT: v_writelane_b32 v32, s87, 29
-; SI-NEXT: v_writelane_b32 v32, s30, 30
-; SI-NEXT: v_writelane_b32 v32, s31, 31
+; SI-NEXT: v_writelane_b32 v32, s30, 28
+; SI-NEXT: v_writelane_b32 v32, s31, 29
; SI-NEXT: v_readfirstlane_b32 s6, v13
; SI-NEXT: v_readfirstlane_b32 s8, v12
; SI-NEXT: v_readfirstlane_b32 s10, v11
@@ -41440,21 +41917,21 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_readfirstlane_b32 s90, v3
; SI-NEXT: v_readfirstlane_b32 s92, v2
; SI-NEXT: v_readfirstlane_b32 s95, v1
-; SI-NEXT: v_readfirstlane_b32 s34, v0
+; SI-NEXT: v_readfirstlane_b32 s30, v0
; SI-NEXT: s_lshr_b32 s94, s29, 16
-; SI-NEXT: s_lshr_b32 s30, s28, 16
-; SI-NEXT: s_lshr_b32 s35, s27, 16
-; SI-NEXT: s_lshr_b32 s69, s26, 16
-; SI-NEXT: s_lshr_b32 s70, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s80, s23, 16
-; SI-NEXT: s_lshr_b32 s81, s22, 16
-; SI-NEXT: s_lshr_b32 s82, s21, 16
-; SI-NEXT: s_lshr_b32 s83, s20, 16
-; SI-NEXT: s_lshr_b32 s84, s19, 16
-; SI-NEXT: s_lshr_b32 s85, s18, 16
-; SI-NEXT: s_lshr_b32 s86, s17, 16
-; SI-NEXT: s_lshr_b32 s87, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_lo, s28, 16
+; SI-NEXT: s_lshr_b32 s31, s27, 16
+; SI-NEXT: s_lshr_b32 s35, s26, 16
+; SI-NEXT: s_lshr_b32 s68, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s70, s23, 16
+; SI-NEXT: s_lshr_b32 s71, s22, 16
+; SI-NEXT: s_lshr_b32 s80, s21, 16
+; SI-NEXT: s_lshr_b32 s81, s20, 16
+; SI-NEXT: s_lshr_b32 s82, s19, 16
+; SI-NEXT: s_lshr_b32 s83, s18, 16
+; SI-NEXT: s_lshr_b32 s84, s17, 16
+; SI-NEXT: s_lshr_b32 s85, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -41467,59 +41944,59 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: s_lshr_b32 s89, s88, 16
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
-; SI-NEXT: s_lshr_b32 s31, s95, 16
-; SI-NEXT: s_lshr_b32 s68, s34, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s95, 16
+; SI-NEXT: s_lshr_b32 s34, s30, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s37, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s70, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s35, 16
+; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s34, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s50, s4, s5
; SI-NEXT: s_and_b32 s4, s95, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s51, s4, s5
; SI-NEXT: s_and_b32 s4, s92, 0xffff
; SI-NEXT: s_lshl_b32 s5, s93, 16
@@ -41557,11 +42034,20 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s63, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s87
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s85
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s84
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -41571,14 +42057,14 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s82
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
@@ -41588,7 +42074,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s81
; SI-NEXT: v_cvt_f32_f16_e32 v6, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
@@ -41597,11 +42083,11 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
; SI-NEXT: v_cvt_f16_f32_e32 v6, v6
; SI-NEXT: v_or_b32_e32 v3, v5, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v5, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v5, s80
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_cvt_f32_f16_e32 v6, s21
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s71
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v6, 0x38000000, v6
@@ -41612,7 +42098,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s70
; SI-NEXT: v_cvt_f32_f16_e32 v9, s23
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
@@ -41621,11 +42107,11 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
; SI-NEXT: v_cvt_f16_f32_e32 v9, v9
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v8, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v8, s69
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_or_b32_e32 v7, v9, v7
; SI-NEXT: v_cvt_f32_f16_e32 v9, s24
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s68
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_add_f32_e32 v9, 0x38000000, v9
@@ -41636,7 +42122,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_or_b32_e32 v8, v9, v8
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v10
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s69
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s35
; SI-NEXT: v_cvt_f32_f16_e32 v12, s26
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
@@ -41645,11 +42131,11 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
; SI-NEXT: v_cvt_f16_f32_e32 v12, v12
; SI-NEXT: v_or_b32_e32 v9, v11, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v11, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v11, s31
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v10, v12, v10
; SI-NEXT: v_cvt_f32_f16_e32 v12, s27
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_lo
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_add_f32_e32 v12, 0x38000000, v12
@@ -41669,11 +42155,11 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_or_b32_e32 v12, v14, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v14, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v14, s34
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; SI-NEXT: v_or_b32_e32 v13, v15, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s34
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v16, vcc_hi
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -41783,11 +42269,8 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v26, v27, v26
; SI-NEXT: v_lshlrev_b32_e32 v27, 16, v28
; SI-NEXT: v_or_b32_e32 v27, v29, v27
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -41820,11 +42303,9 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 30
-; SI-NEXT: v_readlane_b32 s31, v32, 31
-; SI-NEXT: v_readlane_b32 s87, v32, 29
-; SI-NEXT: v_readlane_b32 s86, v32, 28
+; SI-NEXT: .LBB55_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 28
+; SI-NEXT: v_readlane_b32 s31, v32, 29
; SI-NEXT: v_readlane_b32 s85, v32, 27
; SI-NEXT: v_readlane_b32 s84, v32, 26
; SI-NEXT: v_readlane_b32 s83, v32, 25
@@ -41898,12 +42379,9 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
+; VI-NEXT: v_readfirstlane_b32 s6, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
-; VI-NEXT: s_lshr_b32 s15, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v11
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: v_readfirstlane_b32 s6, v13
-; VI-NEXT: s_lshr_b32 s61, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v10
; VI-NEXT: v_readfirstlane_b32 s14, v9
; VI-NEXT: v_readfirstlane_b32 s72, v8
@@ -41911,178 +42389,198 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; VI-NEXT: v_readfirstlane_b32 s76, v6
; VI-NEXT: v_readfirstlane_b32 s79, v5
; VI-NEXT: v_readfirstlane_b32 s91, v4
-; VI-NEXT: v_readfirstlane_b32 s34, v3
-; VI-NEXT: v_readfirstlane_b32 s69, v2
-; VI-NEXT: v_readfirstlane_b32 s80, v1
-; VI-NEXT: v_readfirstlane_b32 s83, v0
-; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_readfirstlane_b32 s30, v3
+; VI-NEXT: v_readfirstlane_b32 s35, v2
+; VI-NEXT: v_readfirstlane_b32 s70, v1
+; VI-NEXT: v_readfirstlane_b32 s81, v0
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s75, s28, 16
+; VI-NEXT: s_lshr_b32 s61, s28, 16
; VI-NEXT: s_lshr_b32 s90, s27, 16
-; VI-NEXT: s_lshr_b32 s31, s26, 16
-; VI-NEXT: s_lshr_b32 s68, s25, 16
-; VI-NEXT: s_lshr_b32 s70, s24, 16
-; VI-NEXT: s_lshr_b32 s81, s23, 16
-; VI-NEXT: s_lshr_b32 s84, s22, 16
-; VI-NEXT: s_lshr_b32 s86, s21, 16
-; VI-NEXT: s_lshr_b32 s87, s20, 16
-; VI-NEXT: s_lshr_b32 s7, s19, 16
-; VI-NEXT: s_lshr_b32 s9, s18, 16
-; VI-NEXT: s_lshr_b32 s11, s17, 16
-; VI-NEXT: s_lshr_b32 s13, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; VI-NEXT: s_lshr_b32 s34, s25, 16
+; VI-NEXT: s_lshr_b32 s68, s24, 16
+; VI-NEXT: s_lshr_b32 s71, s23, 16
+; VI-NEXT: s_lshr_b32 s82, s22, 16
+; VI-NEXT: s_lshr_b32 s84, s21, 16
+; VI-NEXT: s_lshr_b32 s85, s20, 16
+; VI-NEXT: s_lshr_b32 s86, s19, 16
+; VI-NEXT: s_lshr_b32 s87, s18, 16
+; VI-NEXT: s_lshr_b32 s7, s17, 16
+; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s63, s6, 16
-; VI-NEXT: s_lshr_b32 s60, s12, 16
+; VI-NEXT: s_lshr_b32 s11, s8, 16
+; VI-NEXT: s_lshr_b32 s13, s10, 16
+; VI-NEXT: s_lshr_b32 s15, s12, 16
; VI-NEXT: s_lshr_b32 s59, s14, 16
; VI-NEXT: s_lshr_b32 s58, s72, 16
; VI-NEXT: s_lshr_b32 s57, s74, 16
-; VI-NEXT: s_lshr_b32 s62, s76, 16
+; VI-NEXT: s_lshr_b32 s60, s76, 16
; VI-NEXT: s_lshr_b32 s89, s79, 16
-; VI-NEXT: s_lshr_b32 s30, s91, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
-; VI-NEXT: s_lshr_b32 s71, s69, 16
-; VI-NEXT: s_lshr_b32 s82, s80, 16
-; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s91, 16
+; VI-NEXT: s_lshr_b32 s31, s30, 16
+; VI-NEXT: s_lshr_b32 s69, s35, 16
+; VI-NEXT: s_lshr_b32 s80, s70, 16
+; VI-NEXT: s_lshr_b32 s83, s81, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
-; VI-NEXT: v_writelane_b32 v33, s61, 1
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s60, 2
-; VI-NEXT: v_writelane_b32 v33, s59, 3
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: v_writelane_b32 v33, s15, 0
+; VI-NEXT: v_writelane_b32 v33, s59, 1
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s7, 16
+; VI-NEXT: s_lshl_b32 s5, s86, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s11, 16
+; VI-NEXT: s_lshl_b32 s41, s7, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s75, 16
+; VI-NEXT: s_lshl_b32 s5, s61, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s83
-; VI-NEXT: s_lshl_b32 s5, s85, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s81
+; VI-NEXT: s_lshl_b32 s5, s83, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s80
-; VI-NEXT: s_lshl_b32 s5, s82, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s70
+; VI-NEXT: s_lshl_b32 s5, s80, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s69
-; VI-NEXT: s_lshl_b32 s5, s71, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s35
+; VI-NEXT: s_lshl_b32 s5, s69, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s34
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s30
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s53, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s91
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s54, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
; VI-NEXT: s_lshl_b32 s5, s89, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
-; VI-NEXT: s_lshl_b32 s5, s62, 16
-; VI-NEXT: s_mov_b32 s77, s56
+; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_mov_b32 s73, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s75, s57
+; VI-NEXT: s_mov_b32 s78, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s72
; VI-NEXT: s_lshl_b32 s5, s58, 16
-; VI-NEXT: s_mov_b32 s78, s58
+; VI-NEXT: s_mov_b32 s77, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s14
; VI-NEXT: s_lshl_b32 s5, s59, 16
; VI-NEXT: s_or_b32 s59, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s12
-; VI-NEXT: s_lshl_b32 s5, s60, 16
+; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_mov_b32 s88, s60
; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s10
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_mov_b32 s75, s61
; VI-NEXT: s_or_b32 s61, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, s15, 16
-; VI-NEXT: s_mov_b32 s88, s62
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s62, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s63, 16
-; VI-NEXT: s_mov_b32 s15, s63
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b32 s75, s61
+; VI-NEXT: s_mov_b32 s88, s60
+; VI-NEXT: s_mov_b32 s73, s56
+; VI-NEXT: s_mov_b32 s78, s57
+; VI-NEXT: s_mov_b32 s77, s58
+; VI-NEXT: s_mov_b32 s15, s13
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s11, s63
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v27, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s13
-; VI-NEXT: v_mov_b32_e32 v2, s11
+; VI-NEXT: v_mov_b32_e32 v0, s9
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: v_add_f16_sdwa v0, v0, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v27
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v27
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s9
+; VI-NEXT: v_mov_b32_e32 v2, s87
; VI-NEXT: v_add_f16_sdwa v2, v2, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v27
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v3, s86
; VI-NEXT: v_add_f16_sdwa v3, v3, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v27
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s87
+; VI-NEXT: v_mov_b32_e32 v4, s85
; VI-NEXT: v_add_f16_sdwa v4, v4, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v27
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s86
+; VI-NEXT: v_mov_b32_e32 v5, s84
; VI-NEXT: v_add_f16_sdwa v5, v5, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v27
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s84
+; VI-NEXT: v_mov_b32_e32 v6, s82
; VI-NEXT: v_add_f16_sdwa v6, v6, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v27
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s81
+; VI-NEXT: v_mov_b32_e32 v7, s71
; VI-NEXT: v_add_f16_sdwa v7, v7, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v27
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s70
+; VI-NEXT: v_mov_b32_e32 v8, s68
; VI-NEXT: v_add_f16_sdwa v8, v8, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v27
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s68
+; VI-NEXT: v_mov_b32_e32 v9, s34
; VI-NEXT: v_add_f16_sdwa v9, v9, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v27
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s31
+; VI-NEXT: v_mov_b32_e32 v10, vcc_hi
; VI-NEXT: v_add_f16_sdwa v10, v10, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v27
; VI-NEXT: v_or_b32_e32 v10, v11, v10
@@ -42090,31 +42588,31 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v11, v11, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v27
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s73
+; VI-NEXT: v_mov_b32_e32 v12, s75
; VI-NEXT: v_add_f16_sdwa v12, v12, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v27
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s77
+; VI-NEXT: v_mov_b32_e32 v13, s73
; VI-NEXT: v_add_f16_sdwa v13, v13, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v27
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s85
+; VI-NEXT: v_mov_b32_e32 v14, s83
; VI-NEXT: v_add_f16_sdwa v14, v14, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s83, v27
+; VI-NEXT: v_add_f16_e32 v15, s81, v27
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s82
+; VI-NEXT: v_mov_b32_e32 v15, s80
; VI-NEXT: v_add_f16_sdwa v15, v15, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s80, v27
+; VI-NEXT: v_add_f16_e32 v16, s70, v27
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s71
+; VI-NEXT: v_mov_b32_e32 v16, s69
; VI-NEXT: v_add_f16_sdwa v16, v16, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s69, v27
+; VI-NEXT: v_add_f16_e32 v17, s35, v27
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s35
+; VI-NEXT: v_mov_b32_e32 v17, s31
; VI-NEXT: v_add_f16_sdwa v17, v17, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s34, v27
+; VI-NEXT: v_add_f16_e32 v18, s30, v27
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s30
+; VI-NEXT: v_mov_b32_e32 v18, vcc_lo
; VI-NEXT: v_add_f16_sdwa v18, v18, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, s91, v27
; VI-NEXT: v_or_b32_e32 v18, v19, v18
@@ -42126,49 +42624,38 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; VI-NEXT: v_add_f16_sdwa v20, v20, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s76, v27
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s75
+; VI-NEXT: v_mov_b32_e32 v21, s78
; VI-NEXT: v_add_f16_sdwa v21, v21, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s74, v27
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s78
+; VI-NEXT: v_mov_b32_e32 v22, s77
; VI-NEXT: v_add_f16_sdwa v22, v22, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s72, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s14, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s12, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v24, v25, v24
-; VI-NEXT: v_mov_b32_e32 v25, s4
+; VI-NEXT: v_mov_b32_e32 v25, s15
; VI-NEXT: v_add_f16_sdwa v25, v25, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s10, v27
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v25, v26, v25
-; VI-NEXT: v_mov_b32_e32 v26, s4
+; VI-NEXT: v_mov_b32_e32 v26, s13
; VI-NEXT: v_add_f16_sdwa v26, v26, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s8, v27
; VI-NEXT: v_or_b32_e32 v26, v28, v26
-; VI-NEXT: v_mov_b32_e32 v28, s15
+; VI-NEXT: v_mov_b32_e32 v28, s11
; VI-NEXT: v_add_f16_sdwa v28, v28, v27 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s6, v27
; VI-NEXT: v_or_b32_e32 v27, v27, v28
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_mov_b32 s73, s75
-; VI-NEXT: s_mov_b32 s88, s62
-; VI-NEXT: s_mov_b32 s77, s56
-; VI-NEXT: s_mov_b32 s75, s57
-; VI-NEXT: s_mov_b32 s78, s58
-; VI-NEXT: s_mov_b32 s15, s63
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -42201,7 +42688,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
+; VI-NEXT: .LBB55_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -42331,10 +42818,18 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s61, s61, s74
; GFX9-NEXT: s_pack_ll_b32_b16 s62, s62, s73
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s72
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v27, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v27 op_sel_hi:[1,0]
@@ -42364,10 +42859,8 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v25, s61, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v26, s62, v27 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v27, s63, v27 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -42400,7 +42893,7 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
; GFX9-NEXT: v_readlane_b32 s54, v32, 10
; GFX9-NEXT: v_readlane_b32 s53, v32, 9
@@ -42491,11 +42984,16 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s25, s58, s63
; GFX11-NEXT: s_pack_ll_b32_b16 s26, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s44, s57
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -42525,8 +43023,6 @@ define inreg <14 x double> @bitcast_v56f16_to_v14f64_scalar(<56 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v26, 0x200, s26 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v27, 0x200, s27 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -43957,215 +44453,237 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v28, s99, 33
; SI-NEXT: v_writelane_b32 v28, s30, 34
; SI-NEXT: v_writelane_b32 v28, s31, 35
+; SI-NEXT: v_readfirstlane_b32 s87, v4
+; SI-NEXT: v_readfirstlane_b32 s83, v6
+; SI-NEXT: s_lshr_b32 s5, s87, 16
+; SI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
+; SI-NEXT: v_readfirstlane_b32 s98, v8
+; SI-NEXT: s_lshr_b32 s7, s83, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_writelane_b32 v29, s5, 0
; SI-NEXT: v_readfirstlane_b32 s54, v13
; SI-NEXT: v_readfirstlane_b32 s55, v12
; SI-NEXT: v_readfirstlane_b32 s52, v11
; SI-NEXT: v_readfirstlane_b32 s53, v10
; SI-NEXT: v_readfirstlane_b32 s86, v9
-; SI-NEXT: v_readfirstlane_b32 s98, v8
+; SI-NEXT: s_lshr_b32 s9, s98, 16
; SI-NEXT: v_readfirstlane_b32 s96, v7
-; SI-NEXT: v_readfirstlane_b32 s83, v6
; SI-NEXT: v_readfirstlane_b32 s85, v5
-; SI-NEXT: v_readfirstlane_b32 s87, v4
; SI-NEXT: v_readfirstlane_b32 s82, v3
; SI-NEXT: v_readfirstlane_b32 s84, v2
; SI-NEXT: v_readfirstlane_b32 s97, v1
; SI-NEXT: v_readfirstlane_b32 s99, v0
-; SI-NEXT: s_lshr_b32 s64, s29, 16
+; SI-NEXT: v_writelane_b32 v29, s7, 1
+; SI-NEXT: s_lshr_b32 s66, s29, 16
; SI-NEXT: s_lshr_b32 s37, s28, 16
; SI-NEXT: s_lshr_b32 s65, s27, 16
; SI-NEXT: s_lshr_b32 s35, s26, 16
-; SI-NEXT: s_lshr_b32 s50, s25, 16
+; SI-NEXT: s_lshr_b32 s64, s25, 16
; SI-NEXT: s_lshr_b32 s31, s24, 16
; SI-NEXT: s_lshr_b32 s89, s23, 16
-; SI-NEXT: s_lshr_b32 s95, s22, 16
-; SI-NEXT: s_lshr_b32 s77, s21, 16
-; SI-NEXT: s_lshr_b32 s93, s20, 16
-; SI-NEXT: s_lshr_b32 s75, s19, 16
-; SI-NEXT: s_lshr_b32 s92, s18, 16
-; SI-NEXT: s_lshr_b32 s73, s17, 16
-; SI-NEXT: s_lshr_b32 s79, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; SI-NEXT: s_lshr_b32 s79, s21, 16
+; SI-NEXT: s_lshr_b32 s95, s20, 16
+; SI-NEXT: s_lshr_b32 s77, s19, 16
+; SI-NEXT: s_lshr_b32 s93, s18, 16
+; SI-NEXT: s_lshr_b32 s75, s17, 16
+; SI-NEXT: s_lshr_b32 s91, s16, 16
; SI-NEXT: s_lshr_b32 s81, s54, 16
-; SI-NEXT: s_lshr_b32 s11, s55, 16
-; SI-NEXT: s_lshr_b32 s88, s52, 16
-; SI-NEXT: s_lshr_b32 s80, s53, 16
-; SI-NEXT: s_lshr_b32 s68, s86, 16
-; SI-NEXT: s_lshr_b32 s91, s98, 16
+; SI-NEXT: s_lshr_b32 s90, s55, 16
+; SI-NEXT: s_lshr_b32 s80, s52, 16
+; SI-NEXT: s_lshr_b32 s11, s53, 16
+; SI-NEXT: s_lshr_b32 s71, s86, 16
; SI-NEXT: s_lshr_b32 s70, s96, 16
-; SI-NEXT: s_lshr_b32 s67, s83, 16
-; SI-NEXT: s_lshr_b32 s66, s85, 16
-; SI-NEXT: s_lshr_b32 s51, s87, 16
-; SI-NEXT: s_lshr_b32 s71, s82, 16
+; SI-NEXT: s_lshr_b32 s69, s85, 16
+; SI-NEXT: s_lshr_b32 s68, s82, 16
; SI-NEXT: s_lshr_b32 s49, s84, 16
-; SI-NEXT: s_lshr_b32 s69, s97, 16
+; SI-NEXT: s_lshr_b32 s67, s97, 16
; SI-NEXT: s_lshr_b32 s39, s99, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
+; SI-NEXT: v_writelane_b32 v29, s9, 2
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: ; implicit-def: $vgpr29 : SGPR spill to VGPR lane
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v29, s11, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: v_writelane_b32 v29, s11, 3
+; SI-NEXT: v_writelane_b32 v29, s90, 4
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: s_lshl_b32 s42, s5, 16
+; SI-NEXT: s_lshl_b32 s40, s7, 16
; SI-NEXT: s_and_b32 s5, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s73, 16
-; SI-NEXT: s_mov_b32 s9, s73
+; SI-NEXT: s_lshl_b32 s7, s75, 16
; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s75, 16
+; SI-NEXT: s_lshl_b32 s7, s77, 16
; SI-NEXT: s_mov_b32 s78, s75
; SI-NEXT: s_or_b32 s75, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s77, 16
-; SI-NEXT: s_lshl_b32 s72, s79, 16
-; SI-NEXT: s_mov_b32 s90, s79
+; SI-NEXT: s_lshl_b32 s7, s79, 16
+; SI-NEXT: s_mov_b32 s88, s79
; SI-NEXT: s_mov_b32 s79, s77
; SI-NEXT: s_or_b32 s77, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
; SI-NEXT: s_lshl_b32 s7, s89, 16
; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s50, 16
+; SI-NEXT: s_lshl_b32 s7, s64, 16
; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
; SI-NEXT: s_lshl_b32 s7, s65, 16
; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s64, 16
+; SI-NEXT: s_lshl_b32 s7, s66, 16
; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s69, 16
+; SI-NEXT: s_lshl_b32 s7, s67, 16
; SI-NEXT: s_or_b32 s47, s5, s7
; SI-NEXT: s_and_b32 s5, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s71, 16
+; SI-NEXT: s_lshl_b32 s7, s68, 16
; SI-NEXT: s_or_b32 s45, s5, s7
; SI-NEXT: s_and_b32 s5, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s66, 16
+; SI-NEXT: s_lshl_b32 s7, s69, 16
; SI-NEXT: s_or_b32 s43, s5, s7
; SI-NEXT: s_and_b32 s5, s96, 0xffff
; SI-NEXT: s_lshl_b32 s7, s70, 16
; SI-NEXT: s_and_b32 s4, s16, 0xffff
+; SI-NEXT: s_lshl_b32 s72, s91, 16
; SI-NEXT: s_or_b32 s41, s5, s7
; SI-NEXT: s_and_b32 s5, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s68, 16
+; SI-NEXT: s_lshl_b32 s7, s71, 16
; SI-NEXT: s_or_b32 s10, s4, s72
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s74, s92, 16
+; SI-NEXT: s_lshl_b32 s74, s93, 16
; SI-NEXT: s_or_b32 s15, s5, s7
; SI-NEXT: s_and_b32 s5, s52, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s88, 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s11, 16
-; SI-NEXT: s_mov_b32 s11, s73
-; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
+; SI-NEXT: s_lshl_b32 s7, s80, 16
; SI-NEXT: s_or_b32 s8, s4, s74
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s76, s93, 16
+; SI-NEXT: s_lshl_b32 s76, s95, 16
; SI-NEXT: s_or_b32 s13, s5, s7
; SI-NEXT: s_and_b32 s5, s54, 0xffff
; SI-NEXT: s_lshl_b32 s7, s81, 16
-; SI-NEXT: v_writelane_b32 v29, s72, 1
; SI-NEXT: s_or_b32 s6, s4, s76
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s62, s95, 16
-; SI-NEXT: s_or_b32 vcc_hi, s5, s7
-; SI-NEXT: v_writelane_b32 v29, s73, 2
-; SI-NEXT: s_mov_b32 s73, s9
+; SI-NEXT: s_lshl_b32 s62, vcc_hi, 16
+; SI-NEXT: s_lshl_b32 s44, s49, 16
+; SI-NEXT: s_lshl_b32 s14, s9, 16
+; SI-NEXT: s_mov_b32 s50, s49
+; SI-NEXT: s_or_b32 s49, s5, s7
; SI-NEXT: s_mov_b32 s9, s75
; SI-NEXT: s_lshr_b64 s[74:75], s[74:75], 16
; SI-NEXT: s_mov_b32 s7, s77
; SI-NEXT: s_lshr_b64 s[76:77], s[76:77], 16
; SI-NEXT: s_or_b32 s4, s4, s62
; SI-NEXT: s_lshl_b32 s60, s31, 16
+; SI-NEXT: s_lshl_b32 s12, s11, 16
+; SI-NEXT: s_mov_b32 s11, s73
+; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
; SI-NEXT: s_mov_b32 s75, s78
; SI-NEXT: s_mov_b32 s77, s79
; SI-NEXT: s_lshr_b64 s[78:79], s[62:63], 16
; SI-NEXT: s_and_b32 s62, s24, 0xffff
-; SI-NEXT: s_mov_b32 s5, s63
-; SI-NEXT: s_or_b32 s62, s62, s60
-; SI-NEXT: s_mov_b32 s63, s61
-; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v29, s60, 3
; SI-NEXT: s_lshl_b32 s58, s35, 16
-; SI-NEXT: v_writelane_b32 v29, s61, 4
+; SI-NEXT: s_mov_b32 s79, s88
+; SI-NEXT: s_or_b32 s62, s62, s60
+; SI-NEXT: s_mov_b32 s73, s89
+; SI-NEXT: s_lshr_b64 s[88:89], s[60:61], 16
; SI-NEXT: s_and_b32 s60, s26, 0xffff
-; SI-NEXT: s_or_b32 s60, s60, s58
-; SI-NEXT: s_mov_b32 s61, s59
-; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v29, s58, 5
; SI-NEXT: s_lshl_b32 s56, s37, 16
-; SI-NEXT: v_writelane_b32 v29, s59, 6
+; SI-NEXT: s_lshl_b32 s48, s90, 16
+; SI-NEXT: s_mov_b32 s89, s73
+; SI-NEXT: s_or_b32 s60, s60, s58
+; SI-NEXT: s_mov_b32 s73, s91
+; SI-NEXT: s_lshr_b64 s[90:91], s[58:59], 16
; SI-NEXT: s_and_b32 s58, s28, 0xffff
-; SI-NEXT: s_or_b32 s58, s58, s56
-; SI-NEXT: s_mov_b32 s59, s57
-; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
; SI-NEXT: s_lshl_b32 s46, s39, 16
-; SI-NEXT: v_writelane_b32 v29, s56, 7
-; SI-NEXT: s_lshl_b32 s44, s49, 16
-; SI-NEXT: v_writelane_b32 v29, s57, 8
+; SI-NEXT: s_mov_b32 s91, s73
+; SI-NEXT: s_or_b32 s58, s58, s56
+; SI-NEXT: s_mov_b32 s73, s93
+; SI-NEXT: s_lshr_b64 s[92:93], s[56:57], 16
; SI-NEXT: s_and_b32 s56, s99, 0xffff
-; SI-NEXT: s_mov_b32 s30, s95
-; SI-NEXT: s_lshr_b64 s[94:95], s[46:47], 16
-; SI-NEXT: s_lshl_b32 s42, s51, 16
+; SI-NEXT: s_mov_b32 s93, s73
; SI-NEXT: s_or_b32 s56, s56, s46
-; SI-NEXT: s_mov_b32 s95, s30
+; SI-NEXT: s_mov_b32 s73, s95
+; SI-NEXT: s_lshr_b64 s[94:95], s[46:47], 16
; SI-NEXT: s_and_b32 s46, s84, 0xffff
-; SI-NEXT: s_mov_b32 s34, s31
-; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 16
-; SI-NEXT: s_lshl_b32 s40, s67, 16
+; SI-NEXT: s_mov_b32 s95, s73
; SI-NEXT: s_or_b32 s46, s46, s44
-; SI-NEXT: s_mov_b32 s31, s34
+; SI-NEXT: s_mov_b32 s73, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[44:45], 16
; SI-NEXT: s_and_b32 s44, s87, 0xffff
-; SI-NEXT: s_mov_b32 s36, s35
-; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
-; SI-NEXT: s_lshl_b32 s14, s91, 16
+; SI-NEXT: s_mov_b32 vcc_hi, s73
; SI-NEXT: s_or_b32 s44, s44, s42
-; SI-NEXT: s_mov_b32 s35, s36
+; SI-NEXT: s_mov_b32 s73, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[42:43], 16
; SI-NEXT: s_and_b32 s42, s83, 0xffff
-; SI-NEXT: s_mov_b32 s38, s37
-; SI-NEXT: s_lshr_b64 s[36:37], s[40:41], 16
-; SI-NEXT: s_lshl_b32 s12, s80, 16
+; SI-NEXT: s_mov_b32 s31, s73
; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_mov_b32 s37, s38
+; SI-NEXT: s_mov_b32 s73, s35
+; SI-NEXT: s_lshr_b64 s[34:35], s[40:41], 16
; SI-NEXT: s_and_b32 s40, s98, 0xffff
-; SI-NEXT: s_mov_b32 s48, s39
-; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
-; SI-NEXT: s_mov_b32 s72, s80
-; SI-NEXT: s_mov_b32 s80, s70
-; SI-NEXT: s_mov_b32 s70, s68
-; SI-NEXT: s_mov_b32 s68, s66
-; SI-NEXT: s_mov_b32 s66, s19
+; SI-NEXT: s_mov_b32 s35, s73
; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_mov_b32 s39, s48
+; SI-NEXT: s_mov_b32 s73, s37
+; SI-NEXT: s_lshr_b64 s[36:37], s[14:15], 16
; SI-NEXT: s_and_b32 s14, s53, 0xffff
-; SI-NEXT: s_mov_b32 s19, s64
-; SI-NEXT: s_mov_b32 s64, s50
-; SI-NEXT: s_mov_b32 s50, s49
-; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
-; SI-NEXT: s_mov_b32 s79, s90
+; SI-NEXT: s_mov_b32 s37, s73
; SI-NEXT: s_or_b32 s14, s14, s12
-; SI-NEXT: s_mov_b32 s49, s50
-; SI-NEXT: s_mov_b32 s50, s64
-; SI-NEXT: s_mov_b32 s64, s19
-; SI-NEXT: s_mov_b32 s19, s66
-; SI-NEXT: s_mov_b32 s66, s68
-; SI-NEXT: s_mov_b32 s68, s70
-; SI-NEXT: s_mov_b32 s70, s80
+; SI-NEXT: s_mov_b32 s73, s39
+; SI-NEXT: s_lshr_b64 s[38:39], s[12:13], 16
; SI-NEXT: s_and_b32 s12, s55, 0xffff
-; SI-NEXT: s_mov_b32 s80, s91
-; SI-NEXT: s_lshr_b64 s[90:91], vcc, 16
+; SI-NEXT: s_mov_b32 s5, s63
+; SI-NEXT: s_mov_b32 s63, s61
+; SI-NEXT: s_mov_b32 s61, s59
+; SI-NEXT: s_mov_b32 s59, s57
; SI-NEXT: s_mov_b32 s57, s47
; SI-NEXT: s_mov_b32 s47, s45
; SI-NEXT: s_mov_b32 s45, s43
; SI-NEXT: s_mov_b32 s43, s41
; SI-NEXT: s_mov_b32 s41, s15
; SI-NEXT: s_mov_b32 s15, s13
-; SI-NEXT: s_or_b32 s12, s12, vcc_lo
-; SI-NEXT: s_mov_b32 s13, vcc_hi
-; SI-NEXT: s_mov_b32 s91, s80
-; SI-NEXT: s_mov_b32 s80, s72
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_or_b32 s12, s12, s48
+; SI-NEXT: s_mov_b32 s13, s49
+; SI-NEXT: s_lshr_b64 s[48:49], s[48:49], 16
+; SI-NEXT: s_mov_b32 s39, s73
+; SI-NEXT: s_mov_b32 s49, s50
+; SI-NEXT: s_mov_b64 s[50:51], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[50:51], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[50:51], s[50:51], exec
+; SI-NEXT: s_cselect_b32 s73, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s73, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s55, s55, 3
-; SI-NEXT: v_readlane_b32 s5, v29, 0
+; SI-NEXT: v_readlane_b32 s5, v29, 4
; SI-NEXT: s_and_b32 s4, s55, 0xffff
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44175,29 +44693,32 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s53, s53, 3
+; SI-NEXT: v_readlane_b32 s5, v29, 3
; SI-NEXT: s_add_i32 s13, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s53, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s52, s52, 3
; SI-NEXT: s_add_i32 s14, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s52, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s98, s98, 3
+; SI-NEXT: v_readlane_b32 s5, v29, 2
; SI-NEXT: s_add_i32 s15, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s98, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s86, s86, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s86, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s83, s83, 3
+; SI-NEXT: v_readlane_b32 s5, v29, 1
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s83, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s67, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s96, s96, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
@@ -44205,14 +44726,15 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s87, s87, 3
+; SI-NEXT: v_readlane_b32 s5, v29, 0
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s87, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s51, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s85, s85, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s85, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s66, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s84, s84, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
@@ -44222,7 +44744,7 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s82, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s68, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s99, s99, 3
; SI-NEXT: s_add_i32 s47, s4, 0x30000
@@ -44232,7 +44754,7 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s97, s97, 3
; SI-NEXT: s_add_i32 s56, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_lshl_b32 s5, s67, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s28, s28, 3
; SI-NEXT: s_add_i32 s57, s4, 0x30000
@@ -44242,7 +44764,7 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s58, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s64, 16
+; SI-NEXT: s_lshl_b32 s5, s66, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
@@ -44262,12 +44784,12 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
+; SI-NEXT: s_lshl_b32 s5, s64, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s63, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s95, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_and_b32 s5, s23, 0xffff
@@ -44275,161 +44797,145 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s6, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s93, 16
+; SI-NEXT: s_lshl_b32 s7, s95, 16
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s7, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s77, 16
+; SI-NEXT: s_lshl_b32 s8, s79, 16
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_and_b32 s8, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s9, s92, 16
+; SI-NEXT: s_lshl_b32 s9, s93, 16
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s9, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s10, s75, 16
+; SI-NEXT: s_lshl_b32 s10, s77, 16
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_or_b32 s9, s10, s9
; SI-NEXT: s_and_b32 s10, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s11, s79, 16
+; SI-NEXT: s_lshl_b32 s11, s91, 16
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s11, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s73, 16
-; SI-NEXT: s_or_b32 s11, s16, s11
-; SI-NEXT: s_add_i32 s10, s10, 0x30000
-; SI-NEXT: s_add_i32 s11, s11, 0x30000
-; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v29, s16, 1
-; SI-NEXT: v_writelane_b32 v29, s17, 2
-; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v29, s16, 3
-; SI-NEXT: v_writelane_b32 v29, s17, 4
-; SI-NEXT: s_lshr_b64 s[16:17], s[60:61], 16
+; SI-NEXT: s_lshl_b32 s16, s75, 16
+; SI-NEXT: s_add_i32 s4, s4, 0x30000
+; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
; SI-NEXT: s_add_i32 s7, s7, 0x30000
; SI-NEXT: s_add_i32 s8, s8, 0x30000
; SI-NEXT: s_add_i32 s9, s9, 0x30000
-; SI-NEXT: v_writelane_b32 v29, s16, 5
-; SI-NEXT: s_add_i32 s4, s4, 0x30000
-; SI-NEXT: s_add_i32 s5, s5, 0x30000
+; SI-NEXT: s_or_b32 s11, s16, s11
+; SI-NEXT: s_add_i32 s10, s10, 0x30000
+; SI-NEXT: s_add_i32 s11, s11, 0x30000
; SI-NEXT: s_lshr_b64 s[74:75], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[76:77], s[6:7], 16
-; SI-NEXT: v_writelane_b32 v29, s17, 6
-; SI-NEXT: s_lshr_b64 s[16:17], s[58:59], 16
; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v29, s16, 7
+; SI-NEXT: s_lshr_b64 s[88:89], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[90:91], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[58:59], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[56:57], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[46:47], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[48:49], s[14:15], 16
-; SI-NEXT: s_lshr_b64 s[90:91], s[12:13], 16
-; SI-NEXT: s_lshr_b32 s73, s11, 16
-; SI-NEXT: s_lshr_b32 s75, s9, 16
-; SI-NEXT: s_lshr_b32 s77, s7, 16
+; SI-NEXT: s_lshr_b64 vcc, s[46:47], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[12:13], 16
+; SI-NEXT: s_lshr_b32 s75, s11, 16
+; SI-NEXT: s_lshr_b32 s77, s9, 16
+; SI-NEXT: s_lshr_b32 s79, s7, 16
; SI-NEXT: s_lshr_b32 s89, s5, 16
-; SI-NEXT: s_lshr_b32 s50, s63, 16
+; SI-NEXT: s_lshr_b32 s64, s63, 16
; SI-NEXT: s_lshr_b32 s65, s61, 16
-; SI-NEXT: s_lshr_b32 s64, s59, 16
-; SI-NEXT: s_lshr_b32 s69, s57, 16
-; SI-NEXT: s_lshr_b32 s71, s47, 16
-; SI-NEXT: s_lshr_b32 s66, s45, 16
+; SI-NEXT: s_lshr_b32 s66, s59, 16
+; SI-NEXT: s_lshr_b32 s67, s57, 16
+; SI-NEXT: s_lshr_b32 s68, s47, 16
+; SI-NEXT: s_lshr_b32 s69, s45, 16
; SI-NEXT: s_lshr_b32 s70, s43, 16
-; SI-NEXT: s_lshr_b32 s68, s41, 16
-; SI-NEXT: s_lshr_b32 s88, s15, 16
+; SI-NEXT: s_lshr_b32 s71, s41, 16
+; SI-NEXT: s_lshr_b32 s80, s15, 16
; SI-NEXT: s_lshr_b32 s81, s13, 16
-; SI-NEXT: v_writelane_b32 v29, s17, 8
-; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: v_readlane_b32 s16, v29, 1
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_lshl_b32 s16, s72, 16
; SI-NEXT: s_or_b32 s10, s10, s16
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s73, 16
+; SI-NEXT: s_lshl_b32 s16, s75, 16
; SI-NEXT: s_or_b32 s11, s11, s16
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_lshl_b32 s16, s74, 16
; SI-NEXT: s_or_b32 s8, s8, s16
; SI-NEXT: s_and_b32 s9, s9, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s75, 16
+; SI-NEXT: s_lshl_b32 s16, s77, 16
; SI-NEXT: s_or_b32 s9, s9, s16
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_lshl_b32 s16, s76, 16
; SI-NEXT: s_or_b32 s6, s6, s16
; SI-NEXT: s_and_b32 s7, s7, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s77, 16
+; SI-NEXT: s_lshl_b32 s16, s79, 16
; SI-NEXT: s_or_b32 s7, s7, s16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s16, s78, 16
-; SI-NEXT: v_readlane_b32 s17, v29, 2
; SI-NEXT: s_or_b32 s4, s4, s16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_lshl_b32 s16, s89, 16
-; SI-NEXT: v_readlane_b32 s18, v29, 3
; SI-NEXT: s_or_b32 s5, s5, s16
; SI-NEXT: s_and_b32 s16, s62, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s18, 16
-; SI-NEXT: v_readlane_b32 s19, v29, 4
+; SI-NEXT: s_lshl_b32 s17, s88, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s63, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s50, 16
-; SI-NEXT: v_readlane_b32 s20, v29, 5
+; SI-NEXT: s_lshl_b32 s18, s64, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s60, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s20, 16
-; SI-NEXT: v_readlane_b32 s21, v29, 6
+; SI-NEXT: s_lshl_b32 s19, s90, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s61, 0xffff
; SI-NEXT: s_lshl_b32 s20, s65, 16
-; SI-NEXT: v_readlane_b32 s22, v29, 7
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s58, 0xffff
-; SI-NEXT: s_lshl_b32 s21, s22, 16
-; SI-NEXT: v_readlane_b32 s23, v29, 8
+; SI-NEXT: s_lshl_b32 s21, s92, 16
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s59, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s64, 16
+; SI-NEXT: s_lshl_b32 s22, s66, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s56, 0xffff
; SI-NEXT: s_lshl_b32 s23, s94, 16
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s69, 16
+; SI-NEXT: s_lshl_b32 s24, s67, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s30, 16
+; SI-NEXT: s_lshl_b32 s25, vcc_lo, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s47, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s71, 16
+; SI-NEXT: s_lshl_b32 s26, s68, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s44, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s34, 16
+; SI-NEXT: s_lshl_b32 s27, s30, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s28, s66, 16
+; SI-NEXT: s_lshl_b32 s28, s69, 16
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s36, 16
+; SI-NEXT: s_lshl_b32 s29, s34, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s43, 0xffff
; SI-NEXT: s_lshl_b32 s42, s70, 16
; SI-NEXT: s_or_b32 s29, s29, s42
; SI-NEXT: s_and_b32 s40, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s38, 16
+; SI-NEXT: s_lshl_b32 s42, s36, 16
; SI-NEXT: s_or_b32 s40, s40, s42
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s68, 16
+; SI-NEXT: s_lshl_b32 s42, s71, 16
; SI-NEXT: s_or_b32 s41, s41, s42
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s48, 16
+; SI-NEXT: s_lshl_b32 s42, s38, 16
; SI-NEXT: s_or_b32 s14, s14, s42
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s88, 16
+; SI-NEXT: s_lshl_b32 s42, s80, 16
; SI-NEXT: s_or_b32 s15, s15, s42
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s42, s90, 16
+; SI-NEXT: s_lshl_b32 s42, s48, 16
; SI-NEXT: s_or_b32 s12, s12, s42
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_lshl_b32 s42, s81, 16
@@ -44504,44 +45010,6 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: v_writelane_b32 v29, s4, 1
-; SI-NEXT: v_writelane_b32 v29, s5, 2
-; SI-NEXT: v_writelane_b32 v29, s12, 3
-; SI-NEXT: v_writelane_b32 v29, s13, 4
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: v_writelane_b32 v29, s12, 5
-; SI-NEXT: v_writelane_b32 v29, s13, 6
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: v_writelane_b32 v29, s12, 7
-; SI-NEXT: v_writelane_b32 v29, s13, 8
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v56i16_to_v56f16_scalar:
; VI: ; %bb.0:
@@ -44549,10 +45017,8 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_store_dword v28, off, s[0:3], s32 ; 4-byte Folded Spill
; VI-NEXT: s_mov_b64 exec, s[4:5]
-; VI-NEXT: v_writelane_b32 v28, s34, 0
-; VI-NEXT: v_writelane_b32 v28, s35, 1
-; VI-NEXT: v_writelane_b32 v28, s30, 2
-; VI-NEXT: v_writelane_b32 v28, s31, 3
+; VI-NEXT: v_writelane_b32 v28, s30, 0
+; VI-NEXT: v_writelane_b32 v28, s31, 1
; VI-NEXT: v_readfirstlane_b32 s7, v13
; VI-NEXT: v_readfirstlane_b32 s8, v12
; VI-NEXT: v_readfirstlane_b32 s10, v11
@@ -44577,10 +45043,10 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s75, s22, 16
; VI-NEXT: s_lshr_b32 s78, s21, 16
; VI-NEXT: s_lshr_b32 s89, s20, 16
-; VI-NEXT: s_lshr_b32 s30, s19, 16
-; VI-NEXT: s_lshr_b32 s31, s18, 16
-; VI-NEXT: s_lshr_b32 s34, s17, 16
-; VI-NEXT: s_lshr_b32 s35, s16, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; VI-NEXT: s_lshr_b32 s30, s17, 16
+; VI-NEXT: s_lshr_b32 s31, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -44597,18 +45063,26 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s91, s90, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: s_add_i32 s35, s35, 3
+; VI-NEXT: s_add_i32 s31, s31, 3
; VI-NEXT: s_add_i32 s17, s17, 3
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s18, s18, 3
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s19, s19, 3
-; VI-NEXT: s_add_i32 s30, s30, 3
+; VI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -44657,18 +45131,18 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s9, s9, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s34, 16
+; VI-NEXT: s_lshl_b32 s16, s30, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s31, 16
+; VI-NEXT: s_lshl_b32 s17, vcc_hi, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s30, 16
+; VI-NEXT: s_lshl_b32 s18, vcc_lo, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s89, 16
@@ -44742,7 +45216,7 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: s_or_b32 s10, s10, s11
; VI-NEXT: s_or_b32 s8, s8, s9
; VI-NEXT: s_or_b32 s6, s7, s6
-; VI-NEXT: v_readlane_b32 s30, v28, 2
+; VI-NEXT: v_readlane_b32 s30, v28, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -44771,16 +45245,12 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v25, s10
; VI-NEXT: v_mov_b32_e32 v26, s8
; VI-NEXT: v_mov_b32_e32 v27, s6
-; VI-NEXT: v_readlane_b32 s31, v28, 3
-; VI-NEXT: v_readlane_b32 s35, v28, 1
-; VI-NEXT: v_readlane_b32 s34, v28, 0
+; VI-NEXT: v_readlane_b32 s31, v28, 1
; VI-NEXT: s_xor_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v28, off, s[0:3], s32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v56i16_to_v56f16_scalar:
; GFX9: ; %bb.0:
@@ -44837,10 +45307,18 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s74, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v14
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s95, s73
; GFX9-NEXT: v_pk_add_u16 v27, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s94, s72
@@ -44925,10 +45403,8 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v26
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v27, s95
; GFX9-NEXT: v_mov_b32_e32 v26, s94
; GFX9-NEXT: v_mov_b32_e32 v25, s93
@@ -44985,7 +45461,7 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v45, s8
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -45097,11 +45573,16 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s72, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s90, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s90, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s90, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s79, s59
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s76, s57
; GFX11-TRUE16-NEXT: v_pk_add_u16 v27, s59, 3 op_sel_hi:[1,0]
@@ -45187,8 +45668,6 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v26
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v27
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s79 :: v_dual_mov_b32 v26, s89
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s88 :: v_dual_mov_b32 v24, s78
@@ -45295,11 +45774,16 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s72, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s90, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s90, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s90, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s59, s79, s59
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s76, s57
; GFX11-FAKE16-NEXT: v_pk_add_u16 v23, s59, 3 op_sel_hi:[1,0]
@@ -45385,8 +45869,6 @@ define inreg <56 x half> @bitcast_v56i16_to_v56f16_scalar(<56 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 16, v24
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v28, 16, v23
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s79 :: v_dual_mov_b32 v24, s89
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s88 :: v_dual_mov_b32 v26, s78
@@ -46513,10 +46995,18 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s88, s92, 16
; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s95
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s94
@@ -46776,10 +47266,8 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v31, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[26:27], 16
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v52, s72
; SI-NEXT: v_mov_b32_e32 v31, s61
; SI-NEXT: s_waitcnt expcnt(6)
@@ -46840,7 +47328,7 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v32, s59
; SI-NEXT: v_mov_b32_e32 v30, s57
; SI-NEXT: v_mov_b32_e32 v28, s46
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v46
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v0, v2, v0
@@ -46961,10 +47449,8 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 ; 4-byte Folded Spill
-; VI-NEXT: v_writelane_b32 v56, s34, 0
-; VI-NEXT: v_writelane_b32 v56, s35, 1
-; VI-NEXT: v_writelane_b32 v56, s30, 2
-; VI-NEXT: v_writelane_b32 v56, s31, 3
+; VI-NEXT: v_writelane_b32 v56, s30, 0
+; VI-NEXT: v_writelane_b32 v56, s31, 1
; VI-NEXT: v_readfirstlane_b32 s44, v13
; VI-NEXT: v_readfirstlane_b32 s46, v12
; VI-NEXT: v_readfirstlane_b32 s56, v11
@@ -46977,8 +47463,8 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s78, v4
; VI-NEXT: v_readfirstlane_b32 s88, v3
; VI-NEXT: v_readfirstlane_b32 s90, v2
-; VI-NEXT: v_readfirstlane_b32 s30, v1
-; VI-NEXT: v_readfirstlane_b32 s34, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; VI-NEXT: v_readfirstlane_b32 s30, v0
; VI-NEXT: s_lshr_b32 s6, s29, 16
; VI-NEXT: s_lshr_b32 s7, s28, 16
; VI-NEXT: s_lshr_b32 s8, s27, 16
@@ -47005,14 +47491,22 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s89, s88, 16
; VI-NEXT: s_lshr_b32 s91, s90, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; VI-NEXT: s_lshr_b32 s31, s30, 16
-; VI-NEXT: s_lshr_b32 s35, s34, 16
; VI-NEXT: v_readfirstlane_b32 s4, v14
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v28, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v28
; VI-NEXT: v_add_f16_e32 v47, s43, v28
@@ -47042,10 +47536,10 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v51, s7, v28
; VI-NEXT: v_add_f16_e32 v13, s29, v28
; VI-NEXT: v_add_f16_e32 v50, s6, v28
-; VI-NEXT: v_add_f16_e32 v14, s34, v28
-; VI-NEXT: v_add_f16_e32 v49, s35, v28
-; VI-NEXT: v_add_f16_e32 v15, s30, v28
-; VI-NEXT: v_add_f16_e32 v48, s31, v28
+; VI-NEXT: v_add_f16_e32 v14, s30, v28
+; VI-NEXT: v_add_f16_e32 v49, s31, v28
+; VI-NEXT: v_add_f16_e32 v15, vcc_lo, v28
+; VI-NEXT: v_add_f16_e32 v48, vcc_hi, v28
; VI-NEXT: v_add_f16_e32 v16, s90, v28
; VI-NEXT: v_add_f16_e32 v39, s91, v28
; VI-NEXT: v_add_f16_e32 v17, s88, v28
@@ -47070,10 +47564,8 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v29, s47, v28
; VI-NEXT: v_add_f16_e32 v27, s44, v28
; VI-NEXT: v_add_f16_e32 v28, s45, v28
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v28, s45
; VI-NEXT: v_mov_b32_e32 v27, s44
; VI-NEXT: v_mov_b32_e32 v29, s47
@@ -47098,10 +47590,10 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v17, s88
; VI-NEXT: v_mov_b32_e32 v39, s91
; VI-NEXT: v_mov_b32_e32 v16, s90
-; VI-NEXT: v_mov_b32_e32 v48, s31
-; VI-NEXT: v_mov_b32_e32 v15, s30
-; VI-NEXT: v_mov_b32_e32 v49, s35
-; VI-NEXT: v_mov_b32_e32 v14, s34
+; VI-NEXT: v_mov_b32_e32 v48, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v15, vcc_lo
+; VI-NEXT: v_mov_b32_e32 v49, s31
+; VI-NEXT: v_mov_b32_e32 v14, s30
; VI-NEXT: v_mov_b32_e32 v50, s6
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v51, s7
@@ -47130,7 +47622,7 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v47, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v47, 16, v47
; VI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
; VI-NEXT: v_lshlrev_b32_e32 v45, 16, v45
@@ -47175,7 +47667,7 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; VI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; VI-NEXT: v_readlane_b32 s30, v56, 2
+; VI-NEXT: v_readlane_b32 s30, v56, 0
; VI-NEXT: v_or_b32_sdwa v8, v8, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v9, v9, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v10, v10, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -47196,9 +47688,7 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; VI-NEXT: v_or_b32_sdwa v25, v25, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v26, v26, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v28 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v56, 3
-; VI-NEXT: v_readlane_b32 s35, v56, 1
-; VI-NEXT: v_readlane_b32 s34, v56, 0
+; VI-NEXT: v_readlane_b32 s31, v56, 1
; VI-NEXT: s_or_saveexec_b64 s[4:5], -1
; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b64 exec, s[4:5]
@@ -47260,10 +47750,18 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s74, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v14
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s95, s73
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v27, s4, v0 op_sel_hi:[1,0]
@@ -47349,10 +47847,8 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v26
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v27, s95
; GFX9-NEXT: v_mov_b32_e32 v26, s94
; GFX9-NEXT: v_mov_b32_e32 v25, s93
@@ -47409,7 +47905,7 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v45, s8
; GFX9-NEXT: v_mov_b32_e32 v46, s7
; GFX9-NEXT: v_mov_b32_e32 v47, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -47521,11 +48017,16 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s72, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s90, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s90, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s90, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s79, s59
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s57, s76, s57
; GFX11-TRUE16-NEXT: v_pk_add_f16 v27, 0x200, s59 op_sel_hi:[0,1]
@@ -47611,8 +48112,6 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v26
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v27
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s79 :: v_dual_mov_b32 v26, s89
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s88 :: v_dual_mov_b32 v24, s78
@@ -47719,11 +48218,16 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s72, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s90, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s90, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s90
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s90, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s90, s90, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s90, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s90, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s59, s79, s59
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s57, s76, s57
; GFX11-FAKE16-NEXT: v_pk_add_f16 v23, 0x200, s59 op_sel_hi:[0,1]
@@ -47809,8 +48313,6 @@ define inreg <56 x i16> @bitcast_v56f16_to_v56i16_scalar(<56 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v29, 16, v24
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v28, 16, v23
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v23, s79 :: v_dual_mov_b32 v24, s89
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s88 :: v_dual_mov_b32 v26, s78
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index 47a80c7161aae..68e80c289ca18 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -218,10 +218,18 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -252,7 +260,7 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -284,8 +292,6 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v30i32_to_v30f32_scalar:
; VI: ; %bb.0:
@@ -308,10 +314,18 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -342,7 +356,7 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -374,8 +388,6 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v30i32_to_v30f32_scalar:
; GFX9: ; %bb.0:
@@ -398,10 +410,18 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -432,7 +452,7 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -464,8 +484,6 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v30i32_to_v30f32_scalar:
; GFX11: ; %bb.0:
@@ -485,11 +503,16 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -520,7 +543,7 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -538,8 +561,6 @@ define inreg <30 x float> @bitcast_v30i32_to_v30f32_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -786,10 +807,18 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
; SI-NEXT: v_add_f32_e64 v28, s64, 1.0
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -820,10 +849,8 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB3_5
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: s_branch .LBB3_6
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -856,7 +883,7 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB3_5: ; %end
+; SI-NEXT: .LBB3_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -929,10 +956,18 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
; VI-NEXT: v_add_f32_e64 v28, s64, 1.0
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -963,10 +998,8 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB3_5
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: s_branch .LBB3_6
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -999,7 +1032,7 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB3_5: ; %end
+; VI-NEXT: .LBB3_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -1072,10 +1105,18 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX9-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -1106,10 +1147,8 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB3_5
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: s_branch .LBB3_6
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -1142,7 +1181,7 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB3_5: ; %end
+; GFX9-NEXT: .LBB3_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -1216,11 +1255,16 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX11-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -1252,8 +1296,6 @@ define inreg <30 x i32> @bitcast_v30f32_to_v30i32_scalar(<30 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB3_5
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -1520,10 +1562,18 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -1554,7 +1604,7 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -1586,8 +1636,6 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v30i32_to_v15i64_scalar:
; VI: ; %bb.0:
@@ -1610,10 +1658,18 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -1644,7 +1700,7 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -1676,8 +1732,6 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v30i32_to_v15i64_scalar:
; GFX9: ; %bb.0:
@@ -1700,10 +1754,18 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -1734,7 +1796,7 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -1766,8 +1828,6 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v30i32_to_v15i64_scalar:
; GFX11: ; %bb.0:
@@ -1787,11 +1847,16 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB5_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -1822,7 +1887,7 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -1840,8 +1905,6 @@ define inreg <15 x i64> @bitcast_v30i32_to_v15i64_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2079,10 +2142,18 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -2113,7 +2184,7 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2145,8 +2216,6 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v15i64_to_v30i32_scalar:
; VI: ; %bb.0:
@@ -2169,10 +2238,18 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -2203,7 +2280,7 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2235,8 +2312,6 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v15i64_to_v30i32_scalar:
; GFX9: ; %bb.0:
@@ -2259,10 +2334,18 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -2293,7 +2376,7 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2325,8 +2408,6 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v15i64_to_v30i32_scalar:
; GFX11: ; %bb.0:
@@ -2346,11 +2427,16 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB7_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -2381,7 +2467,7 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2399,8 +2485,6 @@ define inreg <30 x i32> @bitcast_v15i64_to_v30i32_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2630,10 +2714,18 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s6, s6, 3
; SI-NEXT: s_add_i32 s7, s7, 3
; SI-NEXT: s_add_i32 s8, s8, 3
@@ -2664,7 +2756,7 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -2696,8 +2788,6 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v30i32_to_v15f64_scalar:
; VI: ; %bb.0:
@@ -2720,10 +2810,18 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -2754,7 +2852,7 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2786,8 +2884,6 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v30i32_to_v15f64_scalar:
; GFX9: ; %bb.0:
@@ -2810,10 +2906,18 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -2844,7 +2948,7 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2876,8 +2980,6 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v30i32_to_v15f64_scalar:
; GFX11: ; %bb.0:
@@ -2897,11 +2999,16 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -2932,7 +3039,7 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -2950,8 +3057,6 @@ define inreg <15 x double> @bitcast_v30i32_to_v15f64_scalar(<30 x i32> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3153,10 +3258,18 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB11_3
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB11_4
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -3172,10 +3285,8 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB11_5
-; SI-NEXT: .LBB11_3:
-; SI-NEXT: s_branch .LBB11_2
-; SI-NEXT: .LBB11_4:
+; SI-NEXT: s_branch .LBB11_6
+; SI-NEXT: .LBB11_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -3208,7 +3319,7 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB11_5: ; %end
+; SI-NEXT: .LBB11_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -3281,10 +3392,18 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -3300,10 +3419,8 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB11_5
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: s_branch .LBB11_6
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -3336,7 +3453,7 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB11_5: ; %end
+; VI-NEXT: .LBB11_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -3409,10 +3526,18 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -3428,10 +3553,8 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB11_5
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: s_branch .LBB11_6
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -3464,7 +3587,7 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB11_5: ; %end
+; GFX9-NEXT: .LBB11_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -3538,11 +3661,16 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB11_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -3559,8 +3687,6 @@ define inreg <30 x i32> @bitcast_v15f64_to_v30i32_scalar(<15 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB11_5
-; GFX11-NEXT: .LBB11_3:
-; GFX11-NEXT: s_branch .LBB11_2
; GFX11-NEXT: .LBB11_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -4566,7 +4692,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s34, s5, 16
; SI-NEXT: s_lshr_b32 s35, s7, 16
@@ -4597,9 +4723,47 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -4659,9 +4823,9 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: .LBB13_3: ; %end
-; SI-NEXT: s_lshl_b32 s45, s30, 16
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: .LBB13_5: ; %end
+; SI-NEXT: s_lshl_b32 s45, vcc_lo, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s45
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -4803,38 +4967,6 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v30i32_to_v60i16_scalar:
; VI: ; %bb.0:
@@ -4846,10 +4978,8 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: v_writelane_b32 v30, s35, 1
; VI-NEXT: v_writelane_b32 v30, s36, 2
; VI-NEXT: v_writelane_b32 v30, s37, 3
-; VI-NEXT: v_writelane_b32 v30, s38, 4
-; VI-NEXT: v_writelane_b32 v30, s39, 5
-; VI-NEXT: v_writelane_b32 v30, s30, 6
-; VI-NEXT: v_writelane_b32 v30, s31, 7
+; VI-NEXT: v_writelane_b32 v30, s30, 4
+; VI-NEXT: v_writelane_b32 v30, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -4868,7 +4998,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -4892,16 +5022,54 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -4954,38 +5122,38 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s39, 16
+; VI-NEXT: s_lshl_b32 s5, s37, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s38, 16
+; VI-NEXT: s_lshl_b32 s16, s36, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s37, 16
+; VI-NEXT: s_lshl_b32 s17, s35, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s36, 16
+; VI-NEXT: s_lshl_b32 s18, s34, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s19, s35, 16
+; VI-NEXT: s_lshl_b32 s19, s31, 16
; VI-NEXT: s_or_b32 s18, s18, s19
; VI-NEXT: s_and_b32 s19, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s20, s34, 16
+; VI-NEXT: s_lshl_b32 s20, s30, 16
; VI-NEXT: s_or_b32 s19, s19, s20
; VI-NEXT: s_and_b32 s20, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s21, s31, 16
+; VI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; VI-NEXT: s_or_b32 s20, s20, s21
; VI-NEXT: s_and_b32 s21, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s22, s30, 16
+; VI-NEXT: s_lshl_b32 s22, vcc_lo, 16
; VI-NEXT: s_or_b32 s21, s21, s22
; VI-NEXT: s_and_b32 s22, 0xffff, s24
; VI-NEXT: s_lshl_b32 s23, s91, 16
@@ -5053,7 +5221,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s44, s46, 16
; VI-NEXT: s_or_b32 s6, s6, s44
-; VI-NEXT: v_readlane_b32 s30, v30, 6
+; VI-NEXT: v_readlane_b32 s30, v30, 4
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -5084,9 +5252,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_readlane_b32 s31, v30, 7
-; VI-NEXT: v_readlane_b32 s39, v30, 5
-; VI-NEXT: v_readlane_b32 s38, v30, 4
+; VI-NEXT: v_readlane_b32 s31, v30, 5
; VI-NEXT: v_readlane_b32 s37, v30, 3
; VI-NEXT: v_readlane_b32 s36, v30, 2
; VI-NEXT: v_readlane_b32 s35, v30, 1
@@ -5096,38 +5262,6 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v30i32_to_v60i16_scalar:
; GFX9: ; %bb.0:
@@ -5135,10 +5269,8 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v30, s34, 0
-; GFX9-NEXT: v_writelane_b32 v30, s35, 1
-; GFX9-NEXT: v_writelane_b32 v30, s30, 2
-; GFX9-NEXT: v_writelane_b32 v30, s31, 3
+; GFX9-NEXT: v_writelane_b32 v30, s30, 0
+; GFX9-NEXT: v_writelane_b32 v30, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -5157,7 +5289,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -5185,12 +5317,50 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -5247,15 +5417,15 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: .LBB13_3: ; %end
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s35
-; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s34
-; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s31
-; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, s30
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: .LBB13_5: ; %end
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s31
+; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s30
+; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, vcc_hi
+; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, vcc_lo
; GFX9-NEXT: s_pack_ll_b32_b16 s18, s20, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s19, s21, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s20, s22, s93
@@ -5282,7 +5452,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s8, s8, s56
; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s47
; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s46
-; GFX9-NEXT: v_readlane_b32 s30, v30, 2
+; GFX9-NEXT: v_readlane_b32 s30, v30, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s16
@@ -5313,46 +5483,12 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_readlane_b32 s31, v30, 3
-; GFX9-NEXT: v_readlane_b32 s35, v30, 1
-; GFX9-NEXT: v_readlane_b32 s34, v30, 0
+; GFX9-NEXT: v_readlane_b32 s31, v30, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v30i32_to_v60i16_scalar:
; GFX11: ; %bb.0:
@@ -5372,7 +5508,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s94, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB13_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s40, s4, 16
; GFX11-NEXT: s_lshr_b32 s41, s5, 16
@@ -5404,9 +5540,46 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB13_3
+; GFX11-NEXT: .LBB13_2:
+; GFX11-NEXT: s_mov_b32 s94, -1
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: .LBB13_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -5467,7 +5640,7 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s93
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s92
@@ -5515,38 +5688,6 @@ define inreg <60 x i16> @bitcast_v30i32_to_v60i16_scalar(<30 x i32> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6949,10 +7090,8 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v30, s87, 29
; SI-NEXT: v_writelane_b32 v30, s96, 30
; SI-NEXT: v_writelane_b32 v30, s97, 31
-; SI-NEXT: v_writelane_b32 v30, s98, 32
-; SI-NEXT: v_writelane_b32 v30, s99, 33
-; SI-NEXT: v_writelane_b32 v30, s30, 34
-; SI-NEXT: v_writelane_b32 v30, s31, 35
+; SI-NEXT: v_writelane_b32 v30, s30, 32
+; SI-NEXT: v_writelane_b32 v30, s31, 33
; SI-NEXT: v_readfirstlane_b32 s7, v15
; SI-NEXT: v_readfirstlane_b32 s9, v14
; SI-NEXT: v_readfirstlane_b32 s11, v13
@@ -6964,25 +7103,25 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s79, v7
; SI-NEXT: v_readfirstlane_b32 s89, v6
; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s35, v3
-; SI-NEXT: v_readfirstlane_b32 s70, v2
-; SI-NEXT: v_readfirstlane_b32 s81, v1
-; SI-NEXT: v_readfirstlane_b32 s84, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s31, v3
+; SI-NEXT: v_readfirstlane_b32 s68, v2
+; SI-NEXT: v_readfirstlane_b32 s71, v1
+; SI-NEXT: v_readfirstlane_b32 s82, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s92, s28, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s68, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s82, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s80, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -6994,47 +7133,47 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s78, s79, 16
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
-; SI-NEXT: s_lshr_b32 s69, s70, 16
-; SI-NEXT: s_lshr_b32 s80, s81, 16
-; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
+; SI-NEXT: s_lshr_b32 s35, s68, 16
+; SI-NEXT: s_lshr_b32 s70, s71, 16
+; SI-NEXT: s_lshr_b32 s81, s82, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_4
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -7045,19 +7184,19 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -7093,61 +7232,70 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB15_3
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
@@ -7164,29 +7312,29 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_add_i32 s68, s68, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -7245,8 +7393,8 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s4, 0x30000
-; SI-NEXT: .LBB15_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v30, 34
+; SI-NEXT: .LBB15_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v30, 32
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -7277,9 +7425,7 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v27, s63
; SI-NEXT: v_mov_b32_e32 v28, s64
; SI-NEXT: v_mov_b32_e32 v29, s65
-; SI-NEXT: v_readlane_b32 s31, v30, 35
-; SI-NEXT: v_readlane_b32 s99, v30, 33
-; SI-NEXT: v_readlane_b32 s98, v30, 32
+; SI-NEXT: v_readlane_b32 s31, v30, 33
; SI-NEXT: v_readlane_b32 s97, v30, 31
; SI-NEXT: v_readlane_b32 s96, v30, 30
; SI-NEXT: v_readlane_b32 s87, v30, 29
@@ -7317,9 +7463,6 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB15_2
;
; VI-LABEL: bitcast_v60i16_to_v30i32_scalar:
; VI: ; %bb.0:
@@ -7360,312 +7503,399 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v30, s87, 29
; VI-NEXT: v_writelane_b32 v30, s30, 30
; VI-NEXT: v_writelane_b32 v30, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s11, v13
-; VI-NEXT: s_lshr_b32 s63, s11, 16
-; VI-NEXT: v_readfirstlane_b32 s13, v12
+; VI-NEXT: s_lshr_b32 s14, s20, 16
; VI-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 s62, s13, 16
-; VI-NEXT: v_readfirstlane_b32 s15, v11
-; VI-NEXT: v_writelane_b32 v31, s63, 0
-; VI-NEXT: s_lshr_b32 s61, s15, 16
-; VI-NEXT: v_readfirstlane_b32 s73, v10
-; VI-NEXT: v_writelane_b32 v31, s62, 1
-; VI-NEXT: v_readfirstlane_b32 s7, v15
-; VI-NEXT: v_readfirstlane_b32 s9, v14
-; VI-NEXT: s_lshr_b32 s60, s73, 16
-; VI-NEXT: v_readfirstlane_b32 s75, v9
+; VI-NEXT: s_lshr_b32 s42, s19, 16
+; VI-NEXT: v_writelane_b32 v31, s14, 0
+; VI-NEXT: s_lshr_b32 s43, s18, 16
+; VI-NEXT: v_readfirstlane_b32 s6, v15
+; VI-NEXT: v_readfirstlane_b32 s8, v14
+; VI-NEXT: v_readfirstlane_b32 s10, v13
+; VI-NEXT: v_readfirstlane_b32 s12, v12
+; VI-NEXT: v_readfirstlane_b32 s87, v11
+; VI-NEXT: v_readfirstlane_b32 s15, v10
+; VI-NEXT: v_readfirstlane_b32 s72, v9
; VI-NEXT: v_readfirstlane_b32 s77, v8
; VI-NEXT: v_readfirstlane_b32 s79, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s31, v5
-; VI-NEXT: v_readfirstlane_b32 s68, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: v_writelane_b32 v31, s61, 2
-; VI-NEXT: s_lshr_b32 s90, s29, 16
-; VI-NEXT: s_lshr_b32 s30, s28, 16
-; VI-NEXT: s_lshr_b32 s34, s27, 16
-; VI-NEXT: s_lshr_b32 s69, s26, 16
-; VI-NEXT: s_lshr_b32 s80, s25, 16
-; VI-NEXT: s_lshr_b32 s83, s24, 16
-; VI-NEXT: s_lshr_b32 s86, s23, 16
-; VI-NEXT: s_lshr_b32 s8, s22, 16
-; VI-NEXT: s_lshr_b32 s10, s21, 16
-; VI-NEXT: s_lshr_b32 s12, s20, 16
-; VI-NEXT: s_lshr_b32 s14, s19, 16
-; VI-NEXT: s_lshr_b32 s72, s18, 16
-; VI-NEXT: s_lshr_b32 s74, s17, 16
-; VI-NEXT: s_lshr_b32 s76, s16, 16
-; VI-NEXT: s_lshr_b32 s66, s7, 16
-; VI-NEXT: s_lshr_b32 s67, s9, 16
-; VI-NEXT: s_lshr_b32 s59, s75, 16
-; VI-NEXT: s_lshr_b32 s58, s77, 16
-; VI-NEXT: s_lshr_b32 s78, s79, 16
-; VI-NEXT: s_lshr_b32 s88, s89, 16
-; VI-NEXT: s_lshr_b32 s91, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
-; VI-NEXT: s_lshr_b32 s81, s82, 16
-; VI-NEXT: s_lshr_b32 s84, s85, 16
-; VI-NEXT: s_lshr_b32 s87, s6, 16
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; VI-NEXT: v_readfirstlane_b32 s34, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v31, s42, 1
+; VI-NEXT: s_lshr_b32 s35, s29, 16
+; VI-NEXT: s_lshr_b32 s68, s28, 16
+; VI-NEXT: s_lshr_b32 s70, s27, 16
+; VI-NEXT: s_lshr_b32 s81, s26, 16
+; VI-NEXT: s_lshr_b32 s84, s25, 16
+; VI-NEXT: s_lshr_b32 s7, s24, 16
+; VI-NEXT: s_lshr_b32 s11, s23, 16
+; VI-NEXT: s_lshr_b32 s73, s22, 16
+; VI-NEXT: s_lshr_b32 s75, s21, 16
+; VI-NEXT: s_lshr_b32 s41, s17, 16
+; VI-NEXT: s_lshr_b32 s5, s16, 16
+; VI-NEXT: s_lshr_b32 s66, s6, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
+; VI-NEXT: s_lshr_b32 s76, s10, 16
+; VI-NEXT: s_lshr_b32 s78, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s87, 16
+; VI-NEXT: s_lshr_b32 s90, s15, 16
+; VI-NEXT: s_lshr_b32 s91, s72, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s77, 16
+; VI-NEXT: s_lshr_b32 s30, s79, 16
+; VI-NEXT: s_lshr_b32 s31, s89, 16
+; VI-NEXT: s_lshr_b32 s67, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s71, s34, 16
+; VI-NEXT: s_lshr_b32 s82, s69, 16
+; VI-NEXT: s_lshr_b32 s85, s80, 16
+; VI-NEXT: s_lshr_b32 s9, s83, 16
+; VI-NEXT: s_lshr_b32 s13, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v31, s60, 3
+; VI-NEXT: v_writelane_b32 v31, s43, 2
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v31, s59, 4
-; VI-NEXT: v_writelane_b32 v31, s58, 5
-; VI-NEXT: s_cbranch_scc0 .LBB15_4
+; VI-NEXT: v_writelane_b32 v31, s41, 3
+; VI-NEXT: v_writelane_b32 v31, s5, 4
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s43, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s42, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s74, 16
+; VI-NEXT: s_lshl_b32 s41, s41, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s77
-; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s75
-; VI-NEXT: s_lshl_b32 s5, s59, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s72
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s73
-; VI-NEXT: s_lshl_b32 s5, s60, 16
-; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s15
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_or_b32 s60, s4, s5
+; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s13
-; VI-NEXT: s_lshl_b32 s5, s62, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s11
-; VI-NEXT: s_lshl_b32 s5, s63, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s10
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s64, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
+; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB15_3
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: v_writelane_b32 v31, s26, 5
+; VI-NEXT: s_mov_b32 s26, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s27
+; VI-NEXT: s_mov_b32 s27, s89
+; VI-NEXT: s_mov_b32 s89, s28
+; VI-NEXT: s_mov_b32 s28, s79
+; VI-NEXT: s_mov_b32 s79, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s77, s13
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: s_mov_b32 s10, s8
+; VI-NEXT: s_mov_b32 s9, s7
+; VI-NEXT: s_mov_b32 s8, s6
+; VI-NEXT: s_mov_b32 s7, s85
+; VI-NEXT: s_mov_b32 s85, s82
+; VI-NEXT: s_mov_b32 s82, s71
+; VI-NEXT: s_mov_b32 s71, s68
+; VI-NEXT: s_mov_b32 s68, s35
+; VI-NEXT: s_mov_b32 s35, s31
+; VI-NEXT: s_mov_b32 s31, s30
+; VI-NEXT: s_mov_b32 s30, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s91
+; VI-NEXT: s_mov_b32 s91, s90
+; VI-NEXT: s_mov_b32 s90, s88
+; VI-NEXT: s_mov_b32 s88, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s66
+; VI-NEXT: s_mov_b32 s6, s75
+; VI-NEXT: s_mov_b32 s75, s72
+; VI-NEXT: s_mov_b32 s72, s73
+; VI-NEXT: s_mov_b32 s73, s15
+; VI-NEXT: s_mov_b32 s15, s87
+; VI-NEXT: s_mov_b32 s87, s84
+; VI-NEXT: s_mov_b32 s84, s81
+; VI-NEXT: s_mov_b32 s81, s70
+; VI-NEXT: s_mov_b32 s70, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s70
+; VI-NEXT: s_mov_b32 s70, s81
+; VI-NEXT: s_mov_b32 s81, s84
+; VI-NEXT: s_mov_b32 s84, s87
+; VI-NEXT: s_mov_b32 s87, s15
+; VI-NEXT: s_mov_b32 s15, s73
+; VI-NEXT: s_mov_b32 s73, s72
+; VI-NEXT: s_mov_b32 s72, s75
+; VI-NEXT: s_mov_b32 s75, s6
+; VI-NEXT: s_mov_b32 s66, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s88
+; VI-NEXT: s_mov_b32 s88, s90
+; VI-NEXT: s_mov_b32 s90, s91
+; VI-NEXT: s_mov_b32 s91, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s30
+; VI-NEXT: s_mov_b32 s30, s31
+; VI-NEXT: s_mov_b32 s31, s35
+; VI-NEXT: s_mov_b32 s35, s68
+; VI-NEXT: s_mov_b32 s68, s71
+; VI-NEXT: s_mov_b32 s71, s82
+; VI-NEXT: s_mov_b32 s82, s85
+; VI-NEXT: s_mov_b32 s85, s7
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_mov_b32 s7, s9
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s11, s13
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s77
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s79
+; VI-NEXT: s_mov_b32 s79, s28
+; VI-NEXT: s_mov_b32 s28, s89
+; VI-NEXT: s_mov_b32 s89, s27
+; VI-NEXT: s_mov_b32 s27, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s26
+; VI-NEXT: v_readlane_b32 s26, v31, 5
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 4
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_add_i32 s17, s17, 3
+; VI-NEXT: v_readlane_b32 s14, v31, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s16, s74, 16
-; VI-NEXT: s_or_b32 s5, s16, s5
+; VI-NEXT: s_lshl_b32 s14, s14, 16
+; VI-NEXT: s_or_b32 s5, s14, s5
+; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_add_i32 s18, s18, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 2
; VI-NEXT: s_add_i32 s36, s4, 0x30000
-; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 1
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 0
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s28, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s48, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s29, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s6, s6, 3
+; VI-NEXT: s_add_i32 s86, s86, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, s86, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s85, s85, 3
+; VI-NEXT: s_add_i32 s83, s83, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s85, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, s83, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s82, s82, 3
+; VI-NEXT: s_add_i32 s80, s80, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s82, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, s80, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s89, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s79, s79, 3
; VI-NEXT: s_add_i32 s56, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s79, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s77, s77, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 5
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s75, s75, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 4
+; VI-NEXT: s_add_i32 s75, s72, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s75, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s73, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 3
+; VI-NEXT: s_add_i32 s73, s15, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s15, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 2
+; VI-NEXT: s_add_i32 s15, s87, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s13, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 1
+; VI-NEXT: s_add_i32 s13, s12, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s11, s11, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 0
+; VI-NEXT: s_add_i32 s11, s10, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s9, 3
+; VI-NEXT: s_add_i32 s9, s8, 3
; VI-NEXT: s_add_i32 s63, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s7, 3
+; VI-NEXT: s_add_i32 s7, s6, 3
; VI-NEXT: s_add_i32 s64, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s65, s4, 0x30000
-; VI-NEXT: .LBB15_3: ; %end
+; VI-NEXT: .LBB15_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v30, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -7734,13 +7964,6 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_4:
-; VI-NEXT: s_mov_b32 vcc_lo, s66
-; VI-NEXT: s_mov_b32 vcc_hi, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, vcc_hi
-; VI-NEXT: s_mov_b32 s66, vcc_lo
-; VI-NEXT: s_branch .LBB15_2
;
; GFX9-LABEL: bitcast_v60i16_to_v30i32_scalar:
; GFX9: ; %bb.0:
@@ -7840,10 +8063,18 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -7874,10 +8105,8 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v28, s64, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v29, s65, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB15_5
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: s_branch .LBB15_6
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -7910,7 +8139,7 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB15_5: ; %end
+; GFX9-NEXT: .LBB15_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -8009,11 +8238,16 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -8045,8 +8279,6 @@ define inreg <30 x i32> @bitcast_v60i16_to_v30i32_scalar(<60 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v28, s28, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v29, s29, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -9033,7 +9265,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s34, s5, 16
; SI-NEXT: s_lshr_b32 s35, s7, 16
@@ -9064,9 +9296,47 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s19, s19, 3
@@ -9126,9 +9396,9 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: .LBB17_3: ; %end
-; SI-NEXT: s_lshl_b32 s45, s30, 16
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: .LBB17_5: ; %end
+; SI-NEXT: s_lshl_b32 s45, vcc_lo, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s45
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -9270,38 +9540,6 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v30i32_to_v60f16_scalar:
; VI: ; %bb.0:
@@ -9313,10 +9551,8 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: v_writelane_b32 v30, s35, 1
; VI-NEXT: v_writelane_b32 v30, s36, 2
; VI-NEXT: v_writelane_b32 v30, s37, 3
-; VI-NEXT: v_writelane_b32 v30, s38, 4
-; VI-NEXT: v_writelane_b32 v30, s39, 5
-; VI-NEXT: v_writelane_b32 v30, s30, 6
-; VI-NEXT: v_writelane_b32 v30, s31, 7
+; VI-NEXT: v_writelane_b32 v30, s30, 4
+; VI-NEXT: v_writelane_b32 v30, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -9335,7 +9571,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -9359,16 +9595,54 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s6, s6, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s8, s8, 3
@@ -9421,38 +9695,38 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s39, 16
+; VI-NEXT: s_lshl_b32 s5, s37, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s38, 16
+; VI-NEXT: s_lshl_b32 s16, s36, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s37, 16
+; VI-NEXT: s_lshl_b32 s17, s35, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s36, 16
+; VI-NEXT: s_lshl_b32 s18, s34, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s19, s35, 16
+; VI-NEXT: s_lshl_b32 s19, s31, 16
; VI-NEXT: s_or_b32 s18, s18, s19
; VI-NEXT: s_and_b32 s19, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s20, s34, 16
+; VI-NEXT: s_lshl_b32 s20, s30, 16
; VI-NEXT: s_or_b32 s19, s19, s20
; VI-NEXT: s_and_b32 s20, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s21, s31, 16
+; VI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; VI-NEXT: s_or_b32 s20, s20, s21
; VI-NEXT: s_and_b32 s21, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s22, s30, 16
+; VI-NEXT: s_lshl_b32 s22, vcc_lo, 16
; VI-NEXT: s_or_b32 s21, s21, s22
; VI-NEXT: s_and_b32 s22, 0xffff, s24
; VI-NEXT: s_lshl_b32 s23, s91, 16
@@ -9520,7 +9794,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s44, s46, 16
; VI-NEXT: s_or_b32 s6, s6, s44
-; VI-NEXT: v_readlane_b32 s30, v30, 6
+; VI-NEXT: v_readlane_b32 s30, v30, 4
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -9551,9 +9825,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_readlane_b32 s31, v30, 7
-; VI-NEXT: v_readlane_b32 s39, v30, 5
-; VI-NEXT: v_readlane_b32 s38, v30, 4
+; VI-NEXT: v_readlane_b32 s31, v30, 5
; VI-NEXT: v_readlane_b32 s37, v30, 3
; VI-NEXT: v_readlane_b32 s36, v30, 2
; VI-NEXT: v_readlane_b32 s35, v30, 1
@@ -9563,38 +9835,6 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v30i32_to_v60f16_scalar:
; GFX9: ; %bb.0:
@@ -9602,10 +9842,8 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v30, s34, 0
-; GFX9-NEXT: v_writelane_b32 v30, s35, 1
-; GFX9-NEXT: v_writelane_b32 v30, s30, 2
-; GFX9-NEXT: v_writelane_b32 v30, s31, 3
+; GFX9-NEXT: v_writelane_b32 v30, s30, 0
+; GFX9-NEXT: v_writelane_b32 v30, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -9624,7 +9862,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -9652,12 +9890,50 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s6, s6, 3
; GFX9-NEXT: s_add_i32 s7, s7, 3
; GFX9-NEXT: s_add_i32 s8, s8, 3
@@ -9714,15 +9990,15 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: .LBB17_3: ; %end
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s35
-; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s34
-; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s31
-; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, s30
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: .LBB17_5: ; %end
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s31
+; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s30
+; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, vcc_hi
+; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, vcc_lo
; GFX9-NEXT: s_pack_ll_b32_b16 s18, s20, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s19, s21, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s20, s22, s93
@@ -9749,7 +10025,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s8, s8, s56
; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s47
; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s46
-; GFX9-NEXT: v_readlane_b32 s30, v30, 2
+; GFX9-NEXT: v_readlane_b32 s30, v30, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s16
@@ -9780,46 +10056,12 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_readlane_b32 s31, v30, 3
-; GFX9-NEXT: v_readlane_b32 s35, v30, 1
-; GFX9-NEXT: v_readlane_b32 s34, v30, 0
+; GFX9-NEXT: v_readlane_b32 s31, v30, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v30i32_to_v60f16_scalar:
; GFX11: ; %bb.0:
@@ -9839,7 +10081,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s94, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB17_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s40, s4, 16
; GFX11-NEXT: s_lshr_b32 s41, s5, 16
@@ -9871,9 +10113,46 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB17_3
+; GFX11-NEXT: .LBB17_2:
+; GFX11-NEXT: s_mov_b32 s94, -1
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: .LBB17_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s4, s4, 3
; GFX11-NEXT: s_add_i32 s5, s5, 3
; GFX11-NEXT: s_add_i32 s6, s6, 3
@@ -9934,7 +10213,7 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s93
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s92
@@ -9982,38 +10261,6 @@ define inreg <60 x half> @bitcast_v30i32_to_v60f16_scalar(<30 x i32> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11560,10 +11807,8 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s87, 29
; SI-NEXT: v_writelane_b32 v32, s96, 30
; SI-NEXT: v_writelane_b32 v32, s97, 31
-; SI-NEXT: v_writelane_b32 v32, s98, 32
-; SI-NEXT: v_writelane_b32 v32, s99, 33
-; SI-NEXT: v_writelane_b32 v32, s30, 34
-; SI-NEXT: v_writelane_b32 v32, s31, 35
+; SI-NEXT: v_writelane_b32 v32, s30, 32
+; SI-NEXT: v_writelane_b32 v32, s31, 33
; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: v_readfirstlane_b32 s8, v14
; SI-NEXT: v_readfirstlane_b32 s10, v13
@@ -11577,23 +11822,23 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v5
; SI-NEXT: v_readfirstlane_b32 s92, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s30, v2
-; SI-NEXT: v_readfirstlane_b32 s35, v1
-; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: s_lshr_b32 s31, s29, 16
-; SI-NEXT: s_lshr_b32 s68, s28, 16
-; SI-NEXT: s_lshr_b32 s71, s27, 16
-; SI-NEXT: s_lshr_b32 s81, s26, 16
-; SI-NEXT: s_lshr_b32 s82, s25, 16
-; SI-NEXT: s_lshr_b32 s83, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v0
+; SI-NEXT: s_lshr_b32 vcc_hi, s29, 16
+; SI-NEXT: s_lshr_b32 s34, s28, 16
+; SI-NEXT: s_lshr_b32 s69, s27, 16
+; SI-NEXT: s_lshr_b32 s71, s26, 16
+; SI-NEXT: s_lshr_b32 s80, s25, 16
+; SI-NEXT: s_lshr_b32 s81, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -11607,63 +11852,63 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s34, s30, 16
-; SI-NEXT: s_lshr_b32 s69, s35, 16
-; SI-NEXT: s_lshr_b32 s80, s70, 16
+; SI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s35, s31, 16
+; SI-NEXT: s_lshr_b32 s70, s68, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_3
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -11704,11 +11949,20 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB19_4
-; SI-NEXT: .LBB19_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s99
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s97
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s96
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -11718,7 +11972,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s87
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -11728,20 +11982,20 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s86
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s84
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -11751,7 +12005,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s83
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -11760,11 +12014,11 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s82
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s81
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
@@ -11775,7 +12029,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s80
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
@@ -11784,11 +12038,11 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s71
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
@@ -11799,7 +12053,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s34
; SI-NEXT: v_cvt_f32_f16_e32 v14, s28
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
@@ -11808,11 +12062,11 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_or_b32_e32 v12, v14, v12
; SI-NEXT: v_cvt_f32_f16_e32 v14, s29
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s70
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -11820,11 +12074,11 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s68
; SI-NEXT: v_or_b32_e32 v13, v14, v13
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s69
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -11832,10 +12086,10 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s30
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v17, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v18, s95
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
@@ -11946,11 +12200,8 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v28, v29, v28
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v30
; SI-NEXT: v_or_b32_e32 v29, v31, v29
-; SI-NEXT: s_branch .LBB19_5
-; SI-NEXT: .LBB19_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB19_2
-; SI-NEXT: .LBB19_4:
+; SI-NEXT: s_branch .LBB19_6
+; SI-NEXT: .LBB19_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -11983,11 +12234,9 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB19_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s31, v32, 35
-; SI-NEXT: v_readlane_b32 s99, v32, 33
-; SI-NEXT: v_readlane_b32 s98, v32, 32
+; SI-NEXT: .LBB19_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 32
+; SI-NEXT: v_readlane_b32 s31, v32, 33
; SI-NEXT: v_readlane_b32 s97, v32, 31
; SI-NEXT: v_readlane_b32 s96, v32, 30
; SI-NEXT: v_readlane_b32 s87, v32, 29
@@ -12065,136 +12314,136 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s6, v15
-; VI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; VI-NEXT: v_readfirstlane_b32 s8, v14
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 vcc_hi, s8, 16
+; VI-NEXT: s_lshr_b32 s75, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v13
-; VI-NEXT: v_writelane_b32 v33, vcc_lo, 0
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_lshr_b32 s63, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v12
-; VI-NEXT: v_writelane_b32 v33, vcc_hi, 1
+; VI-NEXT: v_writelane_b32 v33, s75, 0
; VI-NEXT: s_lshr_b32 s62, s12, 16
; VI-NEXT: v_readfirstlane_b32 s14, v11
-; VI-NEXT: v_writelane_b32 v33, s63, 2
+; VI-NEXT: v_writelane_b32 v33, s63, 1
+; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: s_lshr_b32 s61, s14, 16
; VI-NEXT: v_readfirstlane_b32 s72, v10
-; VI-NEXT: v_writelane_b32 v33, s62, 3
-; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: v_readfirstlane_b32 s74, v9
; VI-NEXT: v_readfirstlane_b32 s76, v8
; VI-NEXT: v_readfirstlane_b32 s78, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s30, v5
-; VI-NEXT: v_readfirstlane_b32 s35, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: v_writelane_b32 v33, s61, 4
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v5
+; VI-NEXT: v_readfirstlane_b32 s31, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v33, s62, 2
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s88, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s68, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s81, s24, 16
-; VI-NEXT: s_lshr_b32 s84, s23, 16
-; VI-NEXT: s_lshr_b32 s86, s22, 16
-; VI-NEXT: s_lshr_b32 s9, s21, 16
-; VI-NEXT: s_lshr_b32 s13, s20, 16
-; VI-NEXT: s_lshr_b32 s15, s19, 16
-; VI-NEXT: s_lshr_b32 s73, s18, 16
-; VI-NEXT: s_lshr_b32 s75, s17, 16
-; VI-NEXT: s_lshr_b32 s77, s16, 16
+; VI-NEXT: s_lshr_b32 s77, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s68, s25, 16
+; VI-NEXT: s_lshr_b32 s71, s24, 16
+; VI-NEXT: s_lshr_b32 s82, s23, 16
+; VI-NEXT: s_lshr_b32 s84, s22, 16
+; VI-NEXT: s_lshr_b32 s87, s21, 16
+; VI-NEXT: s_lshr_b32 s9, s20, 16
+; VI-NEXT: s_lshr_b32 s11, s19, 16
+; VI-NEXT: s_lshr_b32 s13, s18, 16
+; VI-NEXT: s_lshr_b32 s15, s17, 16
+; VI-NEXT: s_lshr_b32 s73, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: s_lshr_b32 s59, s74, 16
; VI-NEXT: s_lshr_b32 s58, s76, 16
; VI-NEXT: s_lshr_b32 s57, s78, 16
-; VI-NEXT: s_lshr_b32 s64, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
-; VI-NEXT: s_lshr_b32 s83, s82, 16
-; VI-NEXT: s_lshr_b32 s87, s85, 16
-; VI-NEXT: s_lshr_b32 s11, s7, 16
+; VI-NEXT: s_lshr_b32 s88, s89, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
+; VI-NEXT: s_lshr_b32 s81, s80, 16
+; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 s7, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v33, s60, 5
+; VI-NEXT: v_writelane_b32 v33, s61, 3
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s59, 6
-; VI-NEXT: v_writelane_b32 v33, s58, 7
-; VI-NEXT: s_cbranch_scc0 .LBB19_3
+; VI-NEXT: v_writelane_b32 v33, s60, 4
+; VI-NEXT: v_writelane_b32 v33, s59, 5
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s77, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s73, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s75, 16
+; VI-NEXT: s_lshl_b32 s41, s15, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s77, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s64, 16
-; VI-NEXT: s_mov_b32 s91, s56
+; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_mov_b32 s90, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s78
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s88, s57
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s77, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s88, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s59, 16
@@ -12212,152 +12461,158 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s63, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
-; VI-NEXT: s_mov_b32 s90, s64
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s64, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_mov_b32 s75, s65
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB19_4
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s90, s56
+; VI-NEXT: s_mov_b32 s77, s57
+; VI-NEXT: s_mov_b32 s88, s58
+; VI-NEXT: s_mov_b32 s75, s65
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v29, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s77
-; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_mov_b32_e32 v0, s73
+; VI-NEXT: v_mov_b32_e32 v2, s15
; VI-NEXT: v_add_f16_sdwa v0, v0, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v29
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v29
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s73
+; VI-NEXT: v_mov_b32_e32 v2, s13
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v29
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_add_f16_sdwa v3, v3, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v29
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s13
+; VI-NEXT: v_mov_b32_e32 v4, s9
; VI-NEXT: v_add_f16_sdwa v4, v4, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v29
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v5, s87
; VI-NEXT: v_add_f16_sdwa v5, v5, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v29
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s86
+; VI-NEXT: v_mov_b32_e32 v6, s84
; VI-NEXT: v_add_f16_sdwa v6, v6, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v29
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s84
+; VI-NEXT: v_mov_b32_e32 v7, s82
; VI-NEXT: v_add_f16_sdwa v7, v7, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v29
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s81
+; VI-NEXT: v_mov_b32_e32 v8, s71
; VI-NEXT: v_add_f16_sdwa v8, v8, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v29
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s70
+; VI-NEXT: v_mov_b32_e32 v9, s68
; VI-NEXT: v_add_f16_sdwa v9, v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v29
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s68
+; VI-NEXT: v_mov_b32_e32 v10, s34
; VI-NEXT: v_add_f16_sdwa v10, v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v29
; VI-NEXT: v_or_b32_e32 v10, v11, v10
-; VI-NEXT: v_mov_b32_e32 v11, s31
+; VI-NEXT: v_mov_b32_e32 v11, vcc_hi
; VI-NEXT: v_add_f16_sdwa v11, v11, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v29
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s79
+; VI-NEXT: v_mov_b32_e32 v12, s91
; VI-NEXT: v_add_f16_sdwa v12, v12, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v29
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s91
+; VI-NEXT: v_mov_b32_e32 v13, s90
; VI-NEXT: v_add_f16_sdwa v13, v13, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v29
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s11
+; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_add_f16_sdwa v14, v14, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s7, v29
+; VI-NEXT: v_add_f16_e32 v15, s86, v29
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s87
+; VI-NEXT: v_mov_b32_e32 v15, s85
; VI-NEXT: v_add_f16_sdwa v15, v15, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s85, v29
+; VI-NEXT: v_add_f16_e32 v16, s83, v29
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s83
+; VI-NEXT: v_mov_b32_e32 v16, s81
; VI-NEXT: v_add_f16_sdwa v16, v16, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s82, v29
+; VI-NEXT: v_add_f16_e32 v17, s80, v29
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s80
+; VI-NEXT: v_mov_b32_e32 v17, s70
; VI-NEXT: v_add_f16_sdwa v17, v17, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s71, v29
+; VI-NEXT: v_add_f16_e32 v18, s69, v29
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s69
+; VI-NEXT: v_mov_b32_e32 v18, s35
; VI-NEXT: v_add_f16_sdwa v18, v18, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v19, s35, v29
+; VI-NEXT: v_add_f16_e32 v19, s31, v29
; VI-NEXT: v_or_b32_e32 v18, v19, v18
-; VI-NEXT: v_mov_b32_e32 v19, s34
+; VI-NEXT: v_mov_b32_e32 v19, s30
; VI-NEXT: v_add_f16_sdwa v19, v19, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v20, s30, v29
+; VI-NEXT: v_add_f16_e32 v20, vcc_lo, v29
; VI-NEXT: v_or_b32_e32 v19, v20, v19
-; VI-NEXT: v_mov_b32_e32 v20, s90
+; VI-NEXT: v_mov_b32_e32 v20, s79
; VI-NEXT: v_add_f16_sdwa v20, v20, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s89, v29
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s88
+; VI-NEXT: v_mov_b32_e32 v21, s77
; VI-NEXT: v_add_f16_sdwa v21, v21, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s78, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 7
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s4
+; VI-NEXT: v_mov_b32_e32 v22, s88
; VI-NEXT: v_add_f16_sdwa v22, v22, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s76, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 6
+; VI-NEXT: v_readlane_b32 s4, v33, 5
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s74, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 5
+; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s72, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 4
+; VI-NEXT: v_readlane_b32 s4, v33, 3
; VI-NEXT: v_or_b32_e32 v24, v25, v24
; VI-NEXT: v_mov_b32_e32 v25, s4
; VI-NEXT: v_add_f16_sdwa v25, v25, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s14, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 2
; VI-NEXT: v_or_b32_e32 v25, v26, v25
; VI-NEXT: v_mov_b32_e32 v26, s4
; VI-NEXT: v_add_f16_sdwa v26, v26, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s12, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v26, v27, v26
; VI-NEXT: v_mov_b32_e32 v27, s4
; VI-NEXT: v_add_f16_sdwa v27, v27, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s10, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 1
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v27, v28, v27
; VI-NEXT: v_mov_b32_e32 v28, s4
; VI-NEXT: v_add_f16_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v30, s8, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v28, v30, v28
-; VI-NEXT: v_mov_b32_e32 v30, s4
+; VI-NEXT: v_mov_b32_e32 v30, s75
; VI-NEXT: v_add_f16_sdwa v30, v30, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v29, s6, v29
; VI-NEXT: v_or_b32_e32 v29, v29, v30
-; VI-NEXT: s_branch .LBB19_5
-; VI-NEXT: .LBB19_3:
-; VI-NEXT: s_mov_b32 s90, s64
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s91, s56
-; VI-NEXT: s_mov_b32 s88, s57
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB19_2
-; VI-NEXT: .LBB19_4:
+; VI-NEXT: s_branch .LBB19_6
+; VI-NEXT: .LBB19_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -12390,7 +12645,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB19_5: ; %end
+; VI-NEXT: .LBB19_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -12528,10 +12783,18 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v29, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v29 op_sel_hi:[1,0]
@@ -12563,10 +12826,8 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v27, s63, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v28, s64, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v29, s65, v29 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB19_5
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: s_branch .LBB19_6
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -12599,7 +12860,7 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB19_5: ; %end
+; GFX9-NEXT: .LBB19_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -12698,11 +12959,16 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -12734,8 +13000,6 @@ define inreg <30 x i32> @bitcast_v60f16_to_v30i32_scalar(<60 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v28, 0x200, s28 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v29, 0x200, s29 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13000,10 +13264,18 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
; SI-NEXT: v_add_f32_e64 v28, s64, 1.0
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -13034,10 +13306,8 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -13070,7 +13340,7 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -13143,10 +13413,18 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
; VI-NEXT: v_add_f32_e64 v28, s64, 1.0
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -13177,10 +13455,8 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -13213,7 +13489,7 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -13286,10 +13562,18 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX9-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -13320,10 +13604,8 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -13356,7 +13638,7 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -13430,11 +13712,16 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB21_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX11-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -13466,8 +13753,6 @@ define inreg <15 x i64> @bitcast_v30f32_to_v15i64_scalar(<30 x float> inreg %a,
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: s_branch .LBB21_2
; GFX11-NEXT: .LBB21_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -13742,10 +14027,18 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
; SI-NEXT: s_add_u32 s9, s9, 3
@@ -13776,7 +14069,7 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s19, s19, 0
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -13808,8 +14101,6 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v15i64_to_v30f32_scalar:
; VI: ; %bb.0:
@@ -13832,10 +14123,18 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -13866,7 +14165,7 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s19, s19, 0
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13898,8 +14197,6 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v15i64_to_v30f32_scalar:
; GFX9: ; %bb.0:
@@ -13922,10 +14219,18 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -13956,7 +14261,7 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s19, s19, 0
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13988,8 +14293,6 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v15i64_to_v30f32_scalar:
; GFX11: ; %bb.0:
@@ -14009,11 +14312,16 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB23_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -14044,7 +14352,7 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s3, s3, 0
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -14062,8 +14370,6 @@ define inreg <30 x float> @bitcast_v15i64_to_v30f32_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14310,10 +14616,18 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v29, s65, 1.0
; SI-NEXT: v_add_f32_e64 v28, s64, 1.0
; SI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -14344,10 +14658,8 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; SI-NEXT: v_add_f32_e64 v2, s38, 1.0
; SI-NEXT: v_add_f32_e64 v1, s37, 1.0
; SI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -14380,7 +14692,7 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -14453,10 +14765,18 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v29, s65, 1.0
; VI-NEXT: v_add_f32_e64 v28, s64, 1.0
; VI-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -14487,10 +14807,8 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; VI-NEXT: v_add_f32_e64 v2, s38, 1.0
; VI-NEXT: v_add_f32_e64 v1, s37, 1.0
; VI-NEXT: v_add_f32_e64 v0, s36, 1.0
-; VI-NEXT: s_branch .LBB25_5
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: s_branch .LBB25_6
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -14523,7 +14841,7 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB25_5: ; %end
+; VI-NEXT: .LBB25_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -14596,10 +14914,18 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX9-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX9-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -14630,10 +14956,8 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; GFX9-NEXT: v_add_f32_e64 v2, s38, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s36, 1.0
-; GFX9-NEXT: s_branch .LBB25_5
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: s_branch .LBB25_6
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -14666,7 +14990,7 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB25_5: ; %end
+; GFX9-NEXT: .LBB25_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -14740,11 +15064,16 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v29, s65, 1.0
; GFX11-NEXT: v_add_f32_e64 v28, s64, 1.0
; GFX11-NEXT: v_add_f32_e64 v27, s63, 1.0
@@ -14776,8 +15105,6 @@ define inreg <15 x double> @bitcast_v30f32_to_v15f64_scalar(<30 x float> inreg %
; GFX11-NEXT: v_add_f32_e64 v1, s37, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s36, 1.0
; GFX11-NEXT: s_branch .LBB25_5
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -15016,10 +15343,18 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB27_3
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB27_4
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -15035,10 +15370,8 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; SI-NEXT: s_branch .LBB27_5
-; SI-NEXT: .LBB27_3:
-; SI-NEXT: s_branch .LBB27_2
-; SI-NEXT: .LBB27_4:
+; SI-NEXT: s_branch .LBB27_6
+; SI-NEXT: .LBB27_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -15071,7 +15404,7 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB27_5: ; %end
+; SI-NEXT: .LBB27_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -15144,10 +15477,18 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -15163,10 +15504,8 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; VI-NEXT: s_branch .LBB27_5
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: s_branch .LBB27_6
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -15199,7 +15538,7 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB27_5: ; %end
+; VI-NEXT: .LBB27_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -15272,10 +15611,18 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -15291,10 +15638,8 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
-; GFX9-NEXT: s_branch .LBB27_5
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: s_branch .LBB27_6
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -15327,7 +15672,7 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB27_5: ; %end
+; GFX9-NEXT: .LBB27_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -15401,11 +15746,16 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB27_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
@@ -15422,8 +15772,6 @@ define inreg <30 x float> @bitcast_v15f64_to_v30f32_scalar(<15 x double> inreg %
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: s_branch .LBB27_5
-; GFX11-NEXT: .LBB27_3:
-; GFX11-NEXT: s_branch .LBB27_2
; GFX11-NEXT: .LBB27_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -16412,7 +16760,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s64, s5, 16
; SI-NEXT: s_lshr_b32 s55, s7, 16
@@ -16429,7 +16777,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: s_lshr_b32 s36, s21, 16
; SI-NEXT: s_lshr_b32 s35, s19, 16
; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
+; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[10:11], 16
@@ -16444,8 +16792,46 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v29, s5, 1.0
; SI-NEXT: v_add_f32_e64 v28, s4, 1.0
; SI-NEXT: v_add_f32_e64 v27, s7, 1.0
@@ -16513,40 +16899,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -16593,7 +16947,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v41, s54
; SI-NEXT: v_mov_b32_e32 v40, s55
; SI-NEXT: v_mov_b32_e32 v55, s64
-; SI-NEXT: v_mov_b32_e32 v30, s30
+; SI-NEXT: v_mov_b32_e32 v30, vcc_lo
; SI-NEXT: v_mov_b32_e32 v31, s44
; SI-NEXT: v_mov_b32_e32 v32, s46
; SI-NEXT: v_mov_b32_e32 v33, s56
@@ -16608,7 +16962,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v51, s90
; SI-NEXT: v_mov_b32_e32 v52, s92
; SI-NEXT: v_mov_b32_e32 v53, s94
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -16758,10 +17112,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_writelane_b32 v60, s35, 1
; VI-NEXT: v_writelane_b32 v60, s36, 2
; VI-NEXT: v_writelane_b32 v60, s37, 3
-; VI-NEXT: v_writelane_b32 v60, s38, 4
-; VI-NEXT: v_writelane_b32 v60, s39, 5
-; VI-NEXT: v_writelane_b32 v60, s30, 6
-; VI-NEXT: v_writelane_b32 v60, s31, 7
+; VI-NEXT: v_writelane_b32 v60, s30, 4
+; VI-NEXT: v_writelane_b32 v60, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -16780,7 +17132,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -16804,16 +17156,54 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v29, s6, 1.0
; VI-NEXT: v_add_f32_e64 v28, s7, 1.0
; VI-NEXT: v_add_f32_e64 v27, s8, 1.0
@@ -16874,40 +17264,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v57, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v58, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; VI-NEXT: s_branch .LBB29_5
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: s_branch .LBB29_6
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -16938,14 +17296,14 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_mov_b32_e32 v59, s39
-; VI-NEXT: v_mov_b32_e32 v58, s38
-; VI-NEXT: v_mov_b32_e32 v57, s37
-; VI-NEXT: v_mov_b32_e32 v56, s36
-; VI-NEXT: v_mov_b32_e32 v47, s35
-; VI-NEXT: v_mov_b32_e32 v46, s34
-; VI-NEXT: v_mov_b32_e32 v45, s31
-; VI-NEXT: v_mov_b32_e32 v44, s30
+; VI-NEXT: v_mov_b32_e32 v59, s37
+; VI-NEXT: v_mov_b32_e32 v58, s36
+; VI-NEXT: v_mov_b32_e32 v57, s35
+; VI-NEXT: v_mov_b32_e32 v56, s34
+; VI-NEXT: v_mov_b32_e32 v47, s31
+; VI-NEXT: v_mov_b32_e32 v46, s30
+; VI-NEXT: v_mov_b32_e32 v45, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v44, vcc_lo
; VI-NEXT: v_mov_b32_e32 v43, s91
; VI-NEXT: v_mov_b32_e32 v42, s90
; VI-NEXT: v_mov_b32_e32 v41, s89
@@ -16968,7 +17326,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v32, s56
; VI-NEXT: v_mov_b32_e32 v31, s47
; VI-NEXT: v_mov_b32_e32 v30, s46
-; VI-NEXT: .LBB29_5: ; %end
+; VI-NEXT: .LBB29_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v59, 16, v59
; VI-NEXT: v_lshlrev_b32_e32 v58, 16, v58
; VI-NEXT: v_lshlrev_b32_e32 v57, 16, v57
@@ -17023,7 +17381,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; VI-NEXT: v_readlane_b32 s30, v60, 6
+; VI-NEXT: v_readlane_b32 s30, v60, 4
; VI-NEXT: v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -17042,9 +17400,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v60, 7
-; VI-NEXT: v_readlane_b32 s39, v60, 5
-; VI-NEXT: v_readlane_b32 s38, v60, 4
+; VI-NEXT: v_readlane_b32 s31, v60, 5
; VI-NEXT: v_readlane_b32 s37, v60, 3
; VI-NEXT: v_readlane_b32 s36, v60, 2
; VI-NEXT: v_readlane_b32 s35, v60, 1
@@ -17073,10 +17429,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -17095,7 +17449,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -17123,12 +17477,50 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v29, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v28, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v27, s8, 1.0
@@ -17189,40 +17581,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v57, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v58, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; GFX9-NEXT: s_branch .LBB29_5
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: s_branch .LBB29_6
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -17253,10 +17613,10 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_mov_b32_e32 v59, s35
-; GFX9-NEXT: v_mov_b32_e32 v58, s34
-; GFX9-NEXT: v_mov_b32_e32 v57, s31
-; GFX9-NEXT: v_mov_b32_e32 v56, s30
+; GFX9-NEXT: v_mov_b32_e32 v59, s31
+; GFX9-NEXT: v_mov_b32_e32 v58, s30
+; GFX9-NEXT: v_mov_b32_e32 v57, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v56, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v47, s95
; GFX9-NEXT: v_mov_b32_e32 v46, s94
; GFX9-NEXT: v_mov_b32_e32 v45, s93
@@ -17283,7 +17643,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v32, s56
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
-; GFX9-NEXT: .LBB29_5: ; %end
+; GFX9-NEXT: .LBB29_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -17338,7 +17698,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
@@ -17357,9 +17717,7 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -17383,11 +17741,11 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s8, 16
@@ -17416,9 +17774,46 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s92, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s93, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-TRUE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB29_3
+; GFX11-TRUE16-NEXT: .LBB29_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v29, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v28, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v27, s6, 1.0
@@ -17479,40 +17874,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB29_5
-; GFX11-TRUE16-NEXT: .LBB29_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: s_branch .LBB29_2
-; GFX11-TRUE16-NEXT: .LBB29_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB29_6
+; GFX11-TRUE16-NEXT: .LBB29_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -17542,8 +17905,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, s56 :: v_dual_mov_b32 v36, s47
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-TRUE16-NEXT: .LBB29_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s41 :: v_dual_mov_b32 v30, s40
+; GFX11-TRUE16-NEXT: .LBB29_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v83.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v82.l
@@ -17594,11 +17957,11 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s14, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB29_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s8, 16
@@ -17627,9 +17990,46 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s92, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s93, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-FAKE16-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB29_3
+; GFX11-FAKE16-NEXT: .LBB29_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: .LBB29_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v25, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v26, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v27, s6, 1.0
@@ -17690,40 +18090,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v82, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v83, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB29_5
-; GFX11-FAKE16-NEXT: .LBB29_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: s_branch .LBB29_2
-; GFX11-FAKE16-NEXT: .LBB29_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB29_6
+; GFX11-FAKE16-NEXT: .LBB29_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v3, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v9, s17
@@ -17753,8 +18121,8 @@ define inreg <60 x i16> @bitcast_v30f32_to_v60i16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s56 :: v_dual_mov_b32 v36, s47
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-FAKE16-NEXT: .LBB29_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s41 :: v_dual_mov_b32 v30, s40
+; GFX11-FAKE16-NEXT: .LBB29_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v84, 0xffff, v1
@@ -19218,10 +19586,8 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v30, s87, 29
; SI-NEXT: v_writelane_b32 v30, s96, 30
; SI-NEXT: v_writelane_b32 v30, s97, 31
-; SI-NEXT: v_writelane_b32 v30, s98, 32
-; SI-NEXT: v_writelane_b32 v30, s99, 33
-; SI-NEXT: v_writelane_b32 v30, s30, 34
-; SI-NEXT: v_writelane_b32 v30, s31, 35
+; SI-NEXT: v_writelane_b32 v30, s30, 32
+; SI-NEXT: v_writelane_b32 v30, s31, 33
; SI-NEXT: v_readfirstlane_b32 s7, v15
; SI-NEXT: v_readfirstlane_b32 s9, v14
; SI-NEXT: v_readfirstlane_b32 s11, v13
@@ -19233,25 +19599,25 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s79, v7
; SI-NEXT: v_readfirstlane_b32 s89, v6
; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s35, v3
-; SI-NEXT: v_readfirstlane_b32 s70, v2
-; SI-NEXT: v_readfirstlane_b32 s81, v1
-; SI-NEXT: v_readfirstlane_b32 s84, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s31, v3
+; SI-NEXT: v_readfirstlane_b32 s68, v2
+; SI-NEXT: v_readfirstlane_b32 s71, v1
+; SI-NEXT: v_readfirstlane_b32 s82, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s92, s28, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s68, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s82, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s80, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -19263,47 +19629,47 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s78, s79, 16
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
-; SI-NEXT: s_lshr_b32 s69, s70, 16
-; SI-NEXT: s_lshr_b32 s80, s81, 16
-; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
+; SI-NEXT: s_lshr_b32 s35, s68, 16
+; SI-NEXT: s_lshr_b32 s70, s71, 16
+; SI-NEXT: s_lshr_b32 s81, s82, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_4
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -19314,19 +19680,19 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -19362,61 +19728,70 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB31_3
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
@@ -19433,29 +19808,29 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_add_i32 s68, s68, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -19514,8 +19889,8 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s4, 0x30000
-; SI-NEXT: .LBB31_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v30, 34
+; SI-NEXT: .LBB31_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v30, 32
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -19546,9 +19921,7 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v27, s63
; SI-NEXT: v_mov_b32_e32 v28, s64
; SI-NEXT: v_mov_b32_e32 v29, s65
-; SI-NEXT: v_readlane_b32 s31, v30, 35
-; SI-NEXT: v_readlane_b32 s99, v30, 33
-; SI-NEXT: v_readlane_b32 s98, v30, 32
+; SI-NEXT: v_readlane_b32 s31, v30, 33
; SI-NEXT: v_readlane_b32 s97, v30, 31
; SI-NEXT: v_readlane_b32 s96, v30, 30
; SI-NEXT: v_readlane_b32 s87, v30, 29
@@ -19586,9 +19959,6 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB31_2
;
; VI-LABEL: bitcast_v60i16_to_v30f32_scalar:
; VI: ; %bb.0:
@@ -19629,312 +19999,399 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v30, s87, 29
; VI-NEXT: v_writelane_b32 v30, s30, 30
; VI-NEXT: v_writelane_b32 v30, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s11, v13
-; VI-NEXT: s_lshr_b32 s63, s11, 16
-; VI-NEXT: v_readfirstlane_b32 s13, v12
+; VI-NEXT: s_lshr_b32 s14, s20, 16
; VI-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 s62, s13, 16
-; VI-NEXT: v_readfirstlane_b32 s15, v11
-; VI-NEXT: v_writelane_b32 v31, s63, 0
-; VI-NEXT: s_lshr_b32 s61, s15, 16
-; VI-NEXT: v_readfirstlane_b32 s73, v10
-; VI-NEXT: v_writelane_b32 v31, s62, 1
-; VI-NEXT: v_readfirstlane_b32 s7, v15
-; VI-NEXT: v_readfirstlane_b32 s9, v14
-; VI-NEXT: s_lshr_b32 s60, s73, 16
-; VI-NEXT: v_readfirstlane_b32 s75, v9
+; VI-NEXT: s_lshr_b32 s42, s19, 16
+; VI-NEXT: v_writelane_b32 v31, s14, 0
+; VI-NEXT: s_lshr_b32 s43, s18, 16
+; VI-NEXT: v_readfirstlane_b32 s6, v15
+; VI-NEXT: v_readfirstlane_b32 s8, v14
+; VI-NEXT: v_readfirstlane_b32 s10, v13
+; VI-NEXT: v_readfirstlane_b32 s12, v12
+; VI-NEXT: v_readfirstlane_b32 s87, v11
+; VI-NEXT: v_readfirstlane_b32 s15, v10
+; VI-NEXT: v_readfirstlane_b32 s72, v9
; VI-NEXT: v_readfirstlane_b32 s77, v8
; VI-NEXT: v_readfirstlane_b32 s79, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s31, v5
-; VI-NEXT: v_readfirstlane_b32 s68, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: v_writelane_b32 v31, s61, 2
-; VI-NEXT: s_lshr_b32 s90, s29, 16
-; VI-NEXT: s_lshr_b32 s30, s28, 16
-; VI-NEXT: s_lshr_b32 s34, s27, 16
-; VI-NEXT: s_lshr_b32 s69, s26, 16
-; VI-NEXT: s_lshr_b32 s80, s25, 16
-; VI-NEXT: s_lshr_b32 s83, s24, 16
-; VI-NEXT: s_lshr_b32 s86, s23, 16
-; VI-NEXT: s_lshr_b32 s8, s22, 16
-; VI-NEXT: s_lshr_b32 s10, s21, 16
-; VI-NEXT: s_lshr_b32 s12, s20, 16
-; VI-NEXT: s_lshr_b32 s14, s19, 16
-; VI-NEXT: s_lshr_b32 s72, s18, 16
-; VI-NEXT: s_lshr_b32 s74, s17, 16
-; VI-NEXT: s_lshr_b32 s76, s16, 16
-; VI-NEXT: s_lshr_b32 s66, s7, 16
-; VI-NEXT: s_lshr_b32 s67, s9, 16
-; VI-NEXT: s_lshr_b32 s59, s75, 16
-; VI-NEXT: s_lshr_b32 s58, s77, 16
-; VI-NEXT: s_lshr_b32 s78, s79, 16
-; VI-NEXT: s_lshr_b32 s88, s89, 16
-; VI-NEXT: s_lshr_b32 s91, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
-; VI-NEXT: s_lshr_b32 s81, s82, 16
-; VI-NEXT: s_lshr_b32 s84, s85, 16
-; VI-NEXT: s_lshr_b32 s87, s6, 16
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; VI-NEXT: v_readfirstlane_b32 s34, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v31, s42, 1
+; VI-NEXT: s_lshr_b32 s35, s29, 16
+; VI-NEXT: s_lshr_b32 s68, s28, 16
+; VI-NEXT: s_lshr_b32 s70, s27, 16
+; VI-NEXT: s_lshr_b32 s81, s26, 16
+; VI-NEXT: s_lshr_b32 s84, s25, 16
+; VI-NEXT: s_lshr_b32 s7, s24, 16
+; VI-NEXT: s_lshr_b32 s11, s23, 16
+; VI-NEXT: s_lshr_b32 s73, s22, 16
+; VI-NEXT: s_lshr_b32 s75, s21, 16
+; VI-NEXT: s_lshr_b32 s41, s17, 16
+; VI-NEXT: s_lshr_b32 s5, s16, 16
+; VI-NEXT: s_lshr_b32 s66, s6, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
+; VI-NEXT: s_lshr_b32 s76, s10, 16
+; VI-NEXT: s_lshr_b32 s78, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s87, 16
+; VI-NEXT: s_lshr_b32 s90, s15, 16
+; VI-NEXT: s_lshr_b32 s91, s72, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s77, 16
+; VI-NEXT: s_lshr_b32 s30, s79, 16
+; VI-NEXT: s_lshr_b32 s31, s89, 16
+; VI-NEXT: s_lshr_b32 s67, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s71, s34, 16
+; VI-NEXT: s_lshr_b32 s82, s69, 16
+; VI-NEXT: s_lshr_b32 s85, s80, 16
+; VI-NEXT: s_lshr_b32 s9, s83, 16
+; VI-NEXT: s_lshr_b32 s13, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v31, s60, 3
+; VI-NEXT: v_writelane_b32 v31, s43, 2
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v31, s59, 4
-; VI-NEXT: v_writelane_b32 v31, s58, 5
-; VI-NEXT: s_cbranch_scc0 .LBB31_4
+; VI-NEXT: v_writelane_b32 v31, s41, 3
+; VI-NEXT: v_writelane_b32 v31, s5, 4
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s43, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s42, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s74, 16
+; VI-NEXT: s_lshl_b32 s41, s41, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s77
-; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s75
-; VI-NEXT: s_lshl_b32 s5, s59, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s72
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s73
-; VI-NEXT: s_lshl_b32 s5, s60, 16
-; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s15
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_or_b32 s60, s4, s5
+; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s13
-; VI-NEXT: s_lshl_b32 s5, s62, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s11
-; VI-NEXT: s_lshl_b32 s5, s63, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s10
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s64, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
+; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB31_3
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: v_writelane_b32 v31, s26, 5
+; VI-NEXT: s_mov_b32 s26, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s27
+; VI-NEXT: s_mov_b32 s27, s89
+; VI-NEXT: s_mov_b32 s89, s28
+; VI-NEXT: s_mov_b32 s28, s79
+; VI-NEXT: s_mov_b32 s79, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s77, s13
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: s_mov_b32 s10, s8
+; VI-NEXT: s_mov_b32 s9, s7
+; VI-NEXT: s_mov_b32 s8, s6
+; VI-NEXT: s_mov_b32 s7, s85
+; VI-NEXT: s_mov_b32 s85, s82
+; VI-NEXT: s_mov_b32 s82, s71
+; VI-NEXT: s_mov_b32 s71, s68
+; VI-NEXT: s_mov_b32 s68, s35
+; VI-NEXT: s_mov_b32 s35, s31
+; VI-NEXT: s_mov_b32 s31, s30
+; VI-NEXT: s_mov_b32 s30, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s91
+; VI-NEXT: s_mov_b32 s91, s90
+; VI-NEXT: s_mov_b32 s90, s88
+; VI-NEXT: s_mov_b32 s88, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s66
+; VI-NEXT: s_mov_b32 s6, s75
+; VI-NEXT: s_mov_b32 s75, s72
+; VI-NEXT: s_mov_b32 s72, s73
+; VI-NEXT: s_mov_b32 s73, s15
+; VI-NEXT: s_mov_b32 s15, s87
+; VI-NEXT: s_mov_b32 s87, s84
+; VI-NEXT: s_mov_b32 s84, s81
+; VI-NEXT: s_mov_b32 s81, s70
+; VI-NEXT: s_mov_b32 s70, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s70
+; VI-NEXT: s_mov_b32 s70, s81
+; VI-NEXT: s_mov_b32 s81, s84
+; VI-NEXT: s_mov_b32 s84, s87
+; VI-NEXT: s_mov_b32 s87, s15
+; VI-NEXT: s_mov_b32 s15, s73
+; VI-NEXT: s_mov_b32 s73, s72
+; VI-NEXT: s_mov_b32 s72, s75
+; VI-NEXT: s_mov_b32 s75, s6
+; VI-NEXT: s_mov_b32 s66, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s88
+; VI-NEXT: s_mov_b32 s88, s90
+; VI-NEXT: s_mov_b32 s90, s91
+; VI-NEXT: s_mov_b32 s91, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s30
+; VI-NEXT: s_mov_b32 s30, s31
+; VI-NEXT: s_mov_b32 s31, s35
+; VI-NEXT: s_mov_b32 s35, s68
+; VI-NEXT: s_mov_b32 s68, s71
+; VI-NEXT: s_mov_b32 s71, s82
+; VI-NEXT: s_mov_b32 s82, s85
+; VI-NEXT: s_mov_b32 s85, s7
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_mov_b32 s7, s9
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s11, s13
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s77
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s79
+; VI-NEXT: s_mov_b32 s79, s28
+; VI-NEXT: s_mov_b32 s28, s89
+; VI-NEXT: s_mov_b32 s89, s27
+; VI-NEXT: s_mov_b32 s27, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s26
+; VI-NEXT: v_readlane_b32 s26, v31, 5
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 4
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_add_i32 s17, s17, 3
+; VI-NEXT: v_readlane_b32 s14, v31, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s16, s74, 16
-; VI-NEXT: s_or_b32 s5, s16, s5
+; VI-NEXT: s_lshl_b32 s14, s14, 16
+; VI-NEXT: s_or_b32 s5, s14, s5
+; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_add_i32 s18, s18, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 2
; VI-NEXT: s_add_i32 s36, s4, 0x30000
-; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 1
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 0
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s28, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s48, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s29, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s6, s6, 3
+; VI-NEXT: s_add_i32 s86, s86, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, s86, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s85, s85, 3
+; VI-NEXT: s_add_i32 s83, s83, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s85, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, s83, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s82, s82, 3
+; VI-NEXT: s_add_i32 s80, s80, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s82, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, s80, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s89, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s79, s79, 3
; VI-NEXT: s_add_i32 s56, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s79, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s77, s77, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 5
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s75, s75, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 4
+; VI-NEXT: s_add_i32 s75, s72, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s75, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s73, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 3
+; VI-NEXT: s_add_i32 s73, s15, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s15, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 2
+; VI-NEXT: s_add_i32 s15, s87, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s13, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 1
+; VI-NEXT: s_add_i32 s13, s12, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s11, s11, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 0
+; VI-NEXT: s_add_i32 s11, s10, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s9, 3
+; VI-NEXT: s_add_i32 s9, s8, 3
; VI-NEXT: s_add_i32 s63, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s7, 3
+; VI-NEXT: s_add_i32 s7, s6, 3
; VI-NEXT: s_add_i32 s64, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s65, s4, 0x30000
-; VI-NEXT: .LBB31_3: ; %end
+; VI-NEXT: .LBB31_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v30, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -20003,13 +20460,6 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_4:
-; VI-NEXT: s_mov_b32 vcc_lo, s66
-; VI-NEXT: s_mov_b32 vcc_hi, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, vcc_hi
-; VI-NEXT: s_mov_b32 s66, vcc_lo
-; VI-NEXT: s_branch .LBB31_2
;
; GFX9-LABEL: bitcast_v60i16_to_v30f32_scalar:
; GFX9: ; %bb.0:
@@ -20109,10 +20559,18 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -20143,10 +20601,8 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v28, s64, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v29, s65, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB31_5
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: s_branch .LBB31_6
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -20179,7 +20635,7 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB31_5: ; %end
+; GFX9-NEXT: .LBB31_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -20278,11 +20734,16 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -20314,8 +20775,6 @@ define inreg <30 x float> @bitcast_v60i16_to_v30f32_scalar(<60 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v28, s28, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v29, s29, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -21285,7 +21744,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s64, s5, 16
; SI-NEXT: s_lshr_b32 s55, s7, 16
@@ -21302,7 +21761,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: s_lshr_b32 s36, s21, 16
; SI-NEXT: s_lshr_b32 s35, s19, 16
; SI-NEXT: s_lshr_b32 s34, s17, 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
+; SI-NEXT: s_lshr_b64 vcc, s[4:5], 16
; SI-NEXT: s_lshr_b64 s[44:45], s[6:7], 16
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
; SI-NEXT: s_lshr_b64 s[56:57], s[10:11], 16
@@ -21317,8 +21776,46 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v29, s5, 1.0
; SI-NEXT: v_add_f32_e64 v28, s4, 1.0
; SI-NEXT: v_add_f32_e64 v27, s7, 1.0
@@ -21386,40 +21883,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -21466,7 +21931,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v41, s54
; SI-NEXT: v_mov_b32_e32 v40, s55
; SI-NEXT: v_mov_b32_e32 v55, s64
-; SI-NEXT: v_mov_b32_e32 v30, s30
+; SI-NEXT: v_mov_b32_e32 v30, vcc_lo
; SI-NEXT: v_mov_b32_e32 v31, s44
; SI-NEXT: v_mov_b32_e32 v32, s46
; SI-NEXT: v_mov_b32_e32 v33, s56
@@ -21481,7 +21946,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v51, s90
; SI-NEXT: v_mov_b32_e32 v52, s92
; SI-NEXT: v_mov_b32_e32 v53, s94
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -21631,10 +22096,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_writelane_b32 v60, s35, 1
; VI-NEXT: v_writelane_b32 v60, s36, 2
; VI-NEXT: v_writelane_b32 v60, s37, 3
-; VI-NEXT: v_writelane_b32 v60, s38, 4
-; VI-NEXT: v_writelane_b32 v60, s39, 5
-; VI-NEXT: v_writelane_b32 v60, s30, 6
-; VI-NEXT: v_writelane_b32 v60, s31, 7
+; VI-NEXT: v_writelane_b32 v60, s30, 4
+; VI-NEXT: v_writelane_b32 v60, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -21653,7 +22116,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -21677,16 +22140,54 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v29, s6, 1.0
; VI-NEXT: v_add_f32_e64 v28, s7, 1.0
; VI-NEXT: v_add_f32_e64 v27, s8, 1.0
@@ -21747,40 +22248,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v57, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v58, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; VI-NEXT: s_branch .LBB33_5
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: s_branch .LBB33_6
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -21811,14 +22280,14 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_mov_b32_e32 v59, s39
-; VI-NEXT: v_mov_b32_e32 v58, s38
-; VI-NEXT: v_mov_b32_e32 v57, s37
-; VI-NEXT: v_mov_b32_e32 v56, s36
-; VI-NEXT: v_mov_b32_e32 v47, s35
-; VI-NEXT: v_mov_b32_e32 v46, s34
-; VI-NEXT: v_mov_b32_e32 v45, s31
-; VI-NEXT: v_mov_b32_e32 v44, s30
+; VI-NEXT: v_mov_b32_e32 v59, s37
+; VI-NEXT: v_mov_b32_e32 v58, s36
+; VI-NEXT: v_mov_b32_e32 v57, s35
+; VI-NEXT: v_mov_b32_e32 v56, s34
+; VI-NEXT: v_mov_b32_e32 v47, s31
+; VI-NEXT: v_mov_b32_e32 v46, s30
+; VI-NEXT: v_mov_b32_e32 v45, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v44, vcc_lo
; VI-NEXT: v_mov_b32_e32 v43, s91
; VI-NEXT: v_mov_b32_e32 v42, s90
; VI-NEXT: v_mov_b32_e32 v41, s89
@@ -21841,7 +22310,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_mov_b32_e32 v32, s56
; VI-NEXT: v_mov_b32_e32 v31, s47
; VI-NEXT: v_mov_b32_e32 v30, s46
-; VI-NEXT: .LBB33_5: ; %end
+; VI-NEXT: .LBB33_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v59, 16, v59
; VI-NEXT: v_lshlrev_b32_e32 v58, 16, v58
; VI-NEXT: v_lshlrev_b32_e32 v57, 16, v57
@@ -21896,7 +22365,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; VI-NEXT: v_readlane_b32 s30, v60, 6
+; VI-NEXT: v_readlane_b32 s30, v60, 4
; VI-NEXT: v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -21915,9 +22384,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; VI-NEXT: v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v60, 7
-; VI-NEXT: v_readlane_b32 s39, v60, 5
-; VI-NEXT: v_readlane_b32 s38, v60, 4
+; VI-NEXT: v_readlane_b32 s31, v60, 5
; VI-NEXT: v_readlane_b32 s37, v60, 3
; VI-NEXT: v_readlane_b32 s36, v60, 2
; VI-NEXT: v_readlane_b32 s35, v60, 1
@@ -21946,10 +22413,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -21968,7 +22433,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -21996,12 +22461,50 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v29, s6, 1.0
; GFX9-NEXT: v_add_f32_e64 v28, s7, 1.0
; GFX9-NEXT: v_add_f32_e64 v27, s8, 1.0
@@ -22062,40 +22565,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v57, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v58, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 16, v0
-; GFX9-NEXT: s_branch .LBB33_5
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: s_branch .LBB33_6
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -22126,10 +22597,10 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_mov_b32_e32 v59, s35
-; GFX9-NEXT: v_mov_b32_e32 v58, s34
-; GFX9-NEXT: v_mov_b32_e32 v57, s31
-; GFX9-NEXT: v_mov_b32_e32 v56, s30
+; GFX9-NEXT: v_mov_b32_e32 v59, s31
+; GFX9-NEXT: v_mov_b32_e32 v58, s30
+; GFX9-NEXT: v_mov_b32_e32 v57, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v56, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v47, s95
; GFX9-NEXT: v_mov_b32_e32 v46, s94
; GFX9-NEXT: v_mov_b32_e32 v45, s93
@@ -22156,7 +22627,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v32, s56
; GFX9-NEXT: v_mov_b32_e32 v31, s47
; GFX9-NEXT: v_mov_b32_e32 v30, s46
-; GFX9-NEXT: .LBB33_5: ; %end
+; GFX9-NEXT: .LBB33_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -22211,7 +22682,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
@@ -22230,9 +22701,7 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -22256,11 +22725,11 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s5, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s4, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s5, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s6, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s44, s7, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s45, s8, 16
@@ -22289,9 +22758,46 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s92, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s93, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-TRUE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB33_3
+; GFX11-TRUE16-NEXT: .LBB33_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v29, s4, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v28, s5, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v27, s6, 1.0
@@ -22352,40 +22858,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB33_5
-; GFX11-TRUE16-NEXT: .LBB33_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: s_branch .LBB33_2
-; GFX11-TRUE16-NEXT: .LBB33_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB33_6
+; GFX11-TRUE16-NEXT: .LBB33_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -22415,8 +22889,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, s56 :: v_dual_mov_b32 v36, s47
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-TRUE16-NEXT: .LBB33_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s41 :: v_dual_mov_b32 v30, s40
+; GFX11-TRUE16-NEXT: .LBB33_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v83.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v82.l
@@ -22467,11 +22941,11 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s14, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB33_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s5, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s5, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s6, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s44, s7, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s45, s8, 16
@@ -22500,9 +22974,46 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s92, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s93, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-FAKE16-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB33_3
+; GFX11-FAKE16-NEXT: .LBB33_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: .LBB33_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f32_e64 v25, s4, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v26, s5, 1.0
; GFX11-FAKE16-NEXT: v_add_f32_e64 v27, s6, 1.0
@@ -22563,40 +23074,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v82, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v83, 16, v4
-; GFX11-FAKE16-NEXT: s_branch .LBB33_5
-; GFX11-FAKE16-NEXT: .LBB33_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: s_branch .LBB33_2
-; GFX11-FAKE16-NEXT: .LBB33_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB33_6
+; GFX11-FAKE16-NEXT: .LBB33_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v3, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v9, s17
@@ -22626,8 +23105,8 @@ define inreg <60 x half> @bitcast_v30f32_to_v60f16_scalar(<30 x float> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, s56 :: v_dual_mov_b32 v36, s47
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, s46 :: v_dual_mov_b32 v34, s45
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, s44 :: v_dual_mov_b32 v32, s43
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s42 :: v_dual_mov_b32 v30, s41
-; GFX11-FAKE16-NEXT: .LBB33_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v31, s41 :: v_dual_mov_b32 v30, s40
+; GFX11-FAKE16-NEXT: .LBB33_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v84, 0xffff, v1
@@ -24235,10 +24714,8 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_writelane_b32 v32, s87, 29
; SI-NEXT: v_writelane_b32 v32, s96, 30
; SI-NEXT: v_writelane_b32 v32, s97, 31
-; SI-NEXT: v_writelane_b32 v32, s98, 32
-; SI-NEXT: v_writelane_b32 v32, s99, 33
-; SI-NEXT: v_writelane_b32 v32, s30, 34
-; SI-NEXT: v_writelane_b32 v32, s31, 35
+; SI-NEXT: v_writelane_b32 v32, s30, 32
+; SI-NEXT: v_writelane_b32 v32, s31, 33
; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: v_readfirstlane_b32 s8, v14
; SI-NEXT: v_readfirstlane_b32 s10, v13
@@ -24252,23 +24729,23 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s90, v5
; SI-NEXT: v_readfirstlane_b32 s92, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s30, v2
-; SI-NEXT: v_readfirstlane_b32 s35, v1
-; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: s_lshr_b32 s31, s29, 16
-; SI-NEXT: s_lshr_b32 s68, s28, 16
-; SI-NEXT: s_lshr_b32 s71, s27, 16
-; SI-NEXT: s_lshr_b32 s81, s26, 16
-; SI-NEXT: s_lshr_b32 s82, s25, 16
-; SI-NEXT: s_lshr_b32 s83, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v0
+; SI-NEXT: s_lshr_b32 vcc_hi, s29, 16
+; SI-NEXT: s_lshr_b32 s34, s28, 16
+; SI-NEXT: s_lshr_b32 s69, s27, 16
+; SI-NEXT: s_lshr_b32 s71, s26, 16
+; SI-NEXT: s_lshr_b32 s80, s25, 16
+; SI-NEXT: s_lshr_b32 s81, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -24282,63 +24759,63 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s34, s30, 16
-; SI-NEXT: s_lshr_b32 s69, s35, 16
-; SI-NEXT: s_lshr_b32 s80, s70, 16
+; SI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s35, s31, 16
+; SI-NEXT: s_lshr_b32 s70, s68, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_3
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -24379,11 +24856,20 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB35_4
-; SI-NEXT: .LBB35_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s99
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s97
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s96
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -24393,7 +24879,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s87
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -24403,20 +24889,20 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s86
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s84
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -24426,7 +24912,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s83
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -24435,11 +24921,11 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s82
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s81
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
@@ -24450,7 +24936,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s80
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
@@ -24459,11 +24945,11 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s71
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
@@ -24474,7 +24960,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s34
; SI-NEXT: v_cvt_f32_f16_e32 v14, s28
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
@@ -24483,11 +24969,11 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_or_b32_e32 v12, v14, v12
; SI-NEXT: v_cvt_f32_f16_e32 v14, s29
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s70
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -24495,11 +24981,11 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s68
; SI-NEXT: v_or_b32_e32 v13, v14, v13
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s69
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -24507,10 +24993,10 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s30
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v17, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v18, s95
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
@@ -24621,11 +25107,8 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_or_b32_e32 v28, v29, v28
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v30
; SI-NEXT: v_or_b32_e32 v29, v31, v29
-; SI-NEXT: s_branch .LBB35_5
-; SI-NEXT: .LBB35_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB35_2
-; SI-NEXT: .LBB35_4:
+; SI-NEXT: s_branch .LBB35_6
+; SI-NEXT: .LBB35_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -24658,11 +25141,9 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB35_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s31, v32, 35
-; SI-NEXT: v_readlane_b32 s99, v32, 33
-; SI-NEXT: v_readlane_b32 s98, v32, 32
+; SI-NEXT: .LBB35_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 32
+; SI-NEXT: v_readlane_b32 s31, v32, 33
; SI-NEXT: v_readlane_b32 s97, v32, 31
; SI-NEXT: v_readlane_b32 s96, v32, 30
; SI-NEXT: v_readlane_b32 s87, v32, 29
@@ -24740,136 +25221,136 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s6, v15
-; VI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; VI-NEXT: v_readfirstlane_b32 s8, v14
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 vcc_hi, s8, 16
+; VI-NEXT: s_lshr_b32 s75, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v13
-; VI-NEXT: v_writelane_b32 v33, vcc_lo, 0
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_lshr_b32 s63, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v12
-; VI-NEXT: v_writelane_b32 v33, vcc_hi, 1
+; VI-NEXT: v_writelane_b32 v33, s75, 0
; VI-NEXT: s_lshr_b32 s62, s12, 16
; VI-NEXT: v_readfirstlane_b32 s14, v11
-; VI-NEXT: v_writelane_b32 v33, s63, 2
+; VI-NEXT: v_writelane_b32 v33, s63, 1
+; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: s_lshr_b32 s61, s14, 16
; VI-NEXT: v_readfirstlane_b32 s72, v10
-; VI-NEXT: v_writelane_b32 v33, s62, 3
-; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: v_readfirstlane_b32 s74, v9
; VI-NEXT: v_readfirstlane_b32 s76, v8
; VI-NEXT: v_readfirstlane_b32 s78, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s30, v5
-; VI-NEXT: v_readfirstlane_b32 s35, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: v_writelane_b32 v33, s61, 4
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v5
+; VI-NEXT: v_readfirstlane_b32 s31, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v33, s62, 2
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s88, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s68, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s81, s24, 16
-; VI-NEXT: s_lshr_b32 s84, s23, 16
-; VI-NEXT: s_lshr_b32 s86, s22, 16
-; VI-NEXT: s_lshr_b32 s9, s21, 16
-; VI-NEXT: s_lshr_b32 s13, s20, 16
-; VI-NEXT: s_lshr_b32 s15, s19, 16
-; VI-NEXT: s_lshr_b32 s73, s18, 16
-; VI-NEXT: s_lshr_b32 s75, s17, 16
-; VI-NEXT: s_lshr_b32 s77, s16, 16
+; VI-NEXT: s_lshr_b32 s77, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s68, s25, 16
+; VI-NEXT: s_lshr_b32 s71, s24, 16
+; VI-NEXT: s_lshr_b32 s82, s23, 16
+; VI-NEXT: s_lshr_b32 s84, s22, 16
+; VI-NEXT: s_lshr_b32 s87, s21, 16
+; VI-NEXT: s_lshr_b32 s9, s20, 16
+; VI-NEXT: s_lshr_b32 s11, s19, 16
+; VI-NEXT: s_lshr_b32 s13, s18, 16
+; VI-NEXT: s_lshr_b32 s15, s17, 16
+; VI-NEXT: s_lshr_b32 s73, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: s_lshr_b32 s59, s74, 16
; VI-NEXT: s_lshr_b32 s58, s76, 16
; VI-NEXT: s_lshr_b32 s57, s78, 16
-; VI-NEXT: s_lshr_b32 s64, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
-; VI-NEXT: s_lshr_b32 s83, s82, 16
-; VI-NEXT: s_lshr_b32 s87, s85, 16
-; VI-NEXT: s_lshr_b32 s11, s7, 16
+; VI-NEXT: s_lshr_b32 s88, s89, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
+; VI-NEXT: s_lshr_b32 s81, s80, 16
+; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 s7, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v33, s60, 5
+; VI-NEXT: v_writelane_b32 v33, s61, 3
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s59, 6
-; VI-NEXT: v_writelane_b32 v33, s58, 7
-; VI-NEXT: s_cbranch_scc0 .LBB35_3
+; VI-NEXT: v_writelane_b32 v33, s60, 4
+; VI-NEXT: v_writelane_b32 v33, s59, 5
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s77, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s73, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s75, 16
+; VI-NEXT: s_lshl_b32 s41, s15, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s77, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s64, 16
-; VI-NEXT: s_mov_b32 s91, s56
+; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_mov_b32 s90, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s78
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s88, s57
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s77, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s88, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s59, 16
@@ -24887,152 +25368,158 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s63, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
-; VI-NEXT: s_mov_b32 s90, s64
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s64, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_mov_b32 s75, s65
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB35_4
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s90, s56
+; VI-NEXT: s_mov_b32 s77, s57
+; VI-NEXT: s_mov_b32 s88, s58
+; VI-NEXT: s_mov_b32 s75, s65
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v29, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s77
-; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_mov_b32_e32 v0, s73
+; VI-NEXT: v_mov_b32_e32 v2, s15
; VI-NEXT: v_add_f16_sdwa v0, v0, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v29
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v29
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s73
+; VI-NEXT: v_mov_b32_e32 v2, s13
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v29
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_add_f16_sdwa v3, v3, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v29
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s13
+; VI-NEXT: v_mov_b32_e32 v4, s9
; VI-NEXT: v_add_f16_sdwa v4, v4, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v29
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v5, s87
; VI-NEXT: v_add_f16_sdwa v5, v5, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v29
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s86
+; VI-NEXT: v_mov_b32_e32 v6, s84
; VI-NEXT: v_add_f16_sdwa v6, v6, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v29
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s84
+; VI-NEXT: v_mov_b32_e32 v7, s82
; VI-NEXT: v_add_f16_sdwa v7, v7, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v29
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s81
+; VI-NEXT: v_mov_b32_e32 v8, s71
; VI-NEXT: v_add_f16_sdwa v8, v8, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v29
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s70
+; VI-NEXT: v_mov_b32_e32 v9, s68
; VI-NEXT: v_add_f16_sdwa v9, v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v29
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s68
+; VI-NEXT: v_mov_b32_e32 v10, s34
; VI-NEXT: v_add_f16_sdwa v10, v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v29
; VI-NEXT: v_or_b32_e32 v10, v11, v10
-; VI-NEXT: v_mov_b32_e32 v11, s31
+; VI-NEXT: v_mov_b32_e32 v11, vcc_hi
; VI-NEXT: v_add_f16_sdwa v11, v11, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v29
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s79
+; VI-NEXT: v_mov_b32_e32 v12, s91
; VI-NEXT: v_add_f16_sdwa v12, v12, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v29
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s91
+; VI-NEXT: v_mov_b32_e32 v13, s90
; VI-NEXT: v_add_f16_sdwa v13, v13, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v29
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s11
+; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_add_f16_sdwa v14, v14, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s7, v29
+; VI-NEXT: v_add_f16_e32 v15, s86, v29
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s87
+; VI-NEXT: v_mov_b32_e32 v15, s85
; VI-NEXT: v_add_f16_sdwa v15, v15, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s85, v29
+; VI-NEXT: v_add_f16_e32 v16, s83, v29
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s83
+; VI-NEXT: v_mov_b32_e32 v16, s81
; VI-NEXT: v_add_f16_sdwa v16, v16, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s82, v29
+; VI-NEXT: v_add_f16_e32 v17, s80, v29
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s80
+; VI-NEXT: v_mov_b32_e32 v17, s70
; VI-NEXT: v_add_f16_sdwa v17, v17, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s71, v29
+; VI-NEXT: v_add_f16_e32 v18, s69, v29
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s69
+; VI-NEXT: v_mov_b32_e32 v18, s35
; VI-NEXT: v_add_f16_sdwa v18, v18, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v19, s35, v29
+; VI-NEXT: v_add_f16_e32 v19, s31, v29
; VI-NEXT: v_or_b32_e32 v18, v19, v18
-; VI-NEXT: v_mov_b32_e32 v19, s34
+; VI-NEXT: v_mov_b32_e32 v19, s30
; VI-NEXT: v_add_f16_sdwa v19, v19, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v20, s30, v29
+; VI-NEXT: v_add_f16_e32 v20, vcc_lo, v29
; VI-NEXT: v_or_b32_e32 v19, v20, v19
-; VI-NEXT: v_mov_b32_e32 v20, s90
+; VI-NEXT: v_mov_b32_e32 v20, s79
; VI-NEXT: v_add_f16_sdwa v20, v20, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s89, v29
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s88
+; VI-NEXT: v_mov_b32_e32 v21, s77
; VI-NEXT: v_add_f16_sdwa v21, v21, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s78, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 7
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s4
+; VI-NEXT: v_mov_b32_e32 v22, s88
; VI-NEXT: v_add_f16_sdwa v22, v22, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s76, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 6
+; VI-NEXT: v_readlane_b32 s4, v33, 5
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s74, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 5
+; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s72, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 4
+; VI-NEXT: v_readlane_b32 s4, v33, 3
; VI-NEXT: v_or_b32_e32 v24, v25, v24
; VI-NEXT: v_mov_b32_e32 v25, s4
; VI-NEXT: v_add_f16_sdwa v25, v25, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s14, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 2
; VI-NEXT: v_or_b32_e32 v25, v26, v25
; VI-NEXT: v_mov_b32_e32 v26, s4
; VI-NEXT: v_add_f16_sdwa v26, v26, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s12, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v26, v27, v26
; VI-NEXT: v_mov_b32_e32 v27, s4
; VI-NEXT: v_add_f16_sdwa v27, v27, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s10, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 1
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v27, v28, v27
; VI-NEXT: v_mov_b32_e32 v28, s4
; VI-NEXT: v_add_f16_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v30, s8, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v28, v30, v28
-; VI-NEXT: v_mov_b32_e32 v30, s4
+; VI-NEXT: v_mov_b32_e32 v30, s75
; VI-NEXT: v_add_f16_sdwa v30, v30, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v29, s6, v29
; VI-NEXT: v_or_b32_e32 v29, v29, v30
-; VI-NEXT: s_branch .LBB35_5
-; VI-NEXT: .LBB35_3:
-; VI-NEXT: s_mov_b32 s90, s64
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s91, s56
-; VI-NEXT: s_mov_b32 s88, s57
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB35_2
-; VI-NEXT: .LBB35_4:
+; VI-NEXT: s_branch .LBB35_6
+; VI-NEXT: .LBB35_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -25065,7 +25552,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB35_5: ; %end
+; VI-NEXT: .LBB35_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -25203,10 +25690,18 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v29, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v29 op_sel_hi:[1,0]
@@ -25238,10 +25733,8 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; GFX9-NEXT: v_pk_add_f16 v27, s63, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v28, s64, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v29, s65, v29 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB35_5
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: s_branch .LBB35_6
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -25274,7 +25767,7 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB35_5: ; %end
+; GFX9-NEXT: .LBB35_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -25373,11 +25866,16 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -25409,8 +25907,6 @@ define inreg <30 x float> @bitcast_v60f16_to_v30f32_scalar(<60 x half> inreg %a,
; GFX11-NEXT: v_pk_add_f16 v28, 0x200, s28 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v29, 0x200, s29 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -25666,10 +26162,18 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s44, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s45, v0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s16, s16, 3
; SI-NEXT: s_addc_u32 s17, s17, 0
; SI-NEXT: s_add_u32 s18, s18, 3
@@ -25700,7 +26204,7 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; SI-NEXT: s_addc_u32 s8, s8, 0
; SI-NEXT: s_add_u32 s7, s7, 3
; SI-NEXT: s_addc_u32 s6, s6, 0
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -25732,8 +26236,6 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; SI-NEXT: v_mov_b32_e32 v28, s7
; SI-NEXT: v_mov_b32_e32 v29, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v15i64_to_v15f64_scalar:
; VI: ; %bb.0:
@@ -25756,10 +26258,18 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s16, s16, 3
; VI-NEXT: s_addc_u32 s17, s17, 0
; VI-NEXT: s_add_u32 s18, s18, 3
@@ -25790,7 +26300,7 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; VI-NEXT: s_addc_u32 s8, s8, 0
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -25822,8 +26332,6 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v15i64_to_v15f64_scalar:
; GFX9: ; %bb.0:
@@ -25846,10 +26354,18 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s16, s16, 3
; GFX9-NEXT: s_addc_u32 s17, s17, 0
; GFX9-NEXT: s_add_u32 s18, s18, 3
@@ -25880,7 +26396,7 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: s_addc_u32 s8, s8, 0
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -25912,8 +26428,6 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v15i64_to_v15f64_scalar:
; GFX11: ; %bb.0:
@@ -25933,11 +26447,16 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s40, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB37_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_u32 s0, s0, 3
; GFX11-NEXT: s_addc_u32 s1, s1, 0
; GFX11-NEXT: s_add_u32 s2, s2, 3
@@ -25968,7 +26487,7 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: s_addc_u32 s6, s6, 0
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_4: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -25985,8 +26504,6 @@ define inreg <15 x double> @bitcast_v15i64_to_v15f64_scalar(<15 x i64> inreg %a,
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -26188,10 +26705,18 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s51, v1
; SI-NEXT: s_cmp_lg_u32 s4, 0
; SI-NEXT: v_readfirstlane_b32 s50, v0
-; SI-NEXT: s_cbranch_scc0 .LBB39_3
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB39_4
-; SI-NEXT: .LBB39_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; SI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; SI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -26207,10 +26732,8 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; SI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; SI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
-; SI-NEXT: s_branch .LBB39_5
-; SI-NEXT: .LBB39_3:
-; SI-NEXT: s_branch .LBB39_2
-; SI-NEXT: .LBB39_4:
+; SI-NEXT: s_branch .LBB39_6
+; SI-NEXT: .LBB39_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -26243,7 +26766,7 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB39_5: ; %end
+; SI-NEXT: .LBB39_6: ; %end
; SI-NEXT: v_readlane_b32 s65, v32, 13
; SI-NEXT: v_readlane_b32 s64, v32, 12
; SI-NEXT: v_readlane_b32 s55, v32, 11
@@ -26316,10 +26839,18 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s51, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s50, v0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; VI-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; VI-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -26335,10 +26866,8 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; VI-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; VI-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -26371,7 +26900,7 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_readlane_b32 s65, v32, 13
; VI-NEXT: v_readlane_b32 s64, v32, 12
; VI-NEXT: v_readlane_b32 s55, v32, 11
@@ -26444,10 +26973,18 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s51, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s50, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX9-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX9-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -26463,10 +27000,8 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_add_f64 v[24:25], s[60:61], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX9-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
-; GFX9-NEXT: s_branch .LBB39_5
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: s_branch .LBB39_6
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -26499,7 +27034,7 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB39_5: ; %end
+; GFX9-NEXT: .LBB39_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -26573,11 +27108,16 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; GFX11-NEXT: s_mov_b32 s37, s1
; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB39_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s0, -1
+; GFX11-NEXT: .LBB39_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB39_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f64 v[0:1], s[36:37], 1.0
; GFX11-NEXT: v_add_f64 v[2:3], s[38:39], 1.0
; GFX11-NEXT: v_add_f64 v[4:5], s[40:41], 1.0
@@ -26594,8 +27134,6 @@ define inreg <15 x i64> @bitcast_v15f64_to_v15i64_scalar(<15 x double> inreg %a,
; GFX11-NEXT: v_add_f64 v[26:27], s[62:63], 1.0
; GFX11-NEXT: v_add_f64 v[28:29], s[64:65], 1.0
; GFX11-NEXT: s_branch .LBB39_5
-; GFX11-NEXT: .LBB39_3:
-; GFX11-NEXT: s_branch .LBB39_2
; GFX11-NEXT: .LBB39_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s36 :: v_dual_mov_b32 v1, s37
; GFX11-NEXT: v_dual_mov_b32 v2, s38 :: v_dual_mov_b32 v3, s39
@@ -27617,7 +28155,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s34, s5, 16
; SI-NEXT: s_lshr_b32 s35, s7, 16
@@ -27648,9 +28186,47 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -27710,9 +28286,9 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: .LBB41_3: ; %end
-; SI-NEXT: s_lshl_b32 s45, s30, 16
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: .LBB41_5: ; %end
+; SI-NEXT: s_lshl_b32 s45, vcc_lo, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s45
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -27854,38 +28430,6 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v15i64_to_v60i16_scalar:
; VI: ; %bb.0:
@@ -27897,10 +28441,8 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: v_writelane_b32 v30, s35, 1
; VI-NEXT: v_writelane_b32 v30, s36, 2
; VI-NEXT: v_writelane_b32 v30, s37, 3
-; VI-NEXT: v_writelane_b32 v30, s38, 4
-; VI-NEXT: v_writelane_b32 v30, s39, 5
-; VI-NEXT: v_writelane_b32 v30, s30, 6
-; VI-NEXT: v_writelane_b32 v30, s31, 7
+; VI-NEXT: v_writelane_b32 v30, s30, 4
+; VI-NEXT: v_writelane_b32 v30, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -27919,7 +28461,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -27943,16 +28485,54 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -28005,38 +28585,38 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s39, 16
+; VI-NEXT: s_lshl_b32 s5, s37, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s38, 16
+; VI-NEXT: s_lshl_b32 s16, s36, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s37, 16
+; VI-NEXT: s_lshl_b32 s17, s35, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s36, 16
+; VI-NEXT: s_lshl_b32 s18, s34, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s19, s35, 16
+; VI-NEXT: s_lshl_b32 s19, s31, 16
; VI-NEXT: s_or_b32 s18, s18, s19
; VI-NEXT: s_and_b32 s19, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s20, s34, 16
+; VI-NEXT: s_lshl_b32 s20, s30, 16
; VI-NEXT: s_or_b32 s19, s19, s20
; VI-NEXT: s_and_b32 s20, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s21, s31, 16
+; VI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; VI-NEXT: s_or_b32 s20, s20, s21
; VI-NEXT: s_and_b32 s21, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s22, s30, 16
+; VI-NEXT: s_lshl_b32 s22, vcc_lo, 16
; VI-NEXT: s_or_b32 s21, s21, s22
; VI-NEXT: s_and_b32 s22, 0xffff, s24
; VI-NEXT: s_lshl_b32 s23, s91, 16
@@ -28104,7 +28684,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s44, s46, 16
; VI-NEXT: s_or_b32 s6, s6, s44
-; VI-NEXT: v_readlane_b32 s30, v30, 6
+; VI-NEXT: v_readlane_b32 s30, v30, 4
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -28135,9 +28715,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_readlane_b32 s31, v30, 7
-; VI-NEXT: v_readlane_b32 s39, v30, 5
-; VI-NEXT: v_readlane_b32 s38, v30, 4
+; VI-NEXT: v_readlane_b32 s31, v30, 5
; VI-NEXT: v_readlane_b32 s37, v30, 3
; VI-NEXT: v_readlane_b32 s36, v30, 2
; VI-NEXT: v_readlane_b32 s35, v30, 1
@@ -28147,38 +28725,6 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v15i64_to_v60i16_scalar:
; GFX9: ; %bb.0:
@@ -28186,10 +28732,8 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v30, s34, 0
-; GFX9-NEXT: v_writelane_b32 v30, s35, 1
-; GFX9-NEXT: v_writelane_b32 v30, s30, 2
-; GFX9-NEXT: v_writelane_b32 v30, s31, 3
+; GFX9-NEXT: v_writelane_b32 v30, s30, 0
+; GFX9-NEXT: v_writelane_b32 v30, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -28208,7 +28752,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -28236,12 +28780,50 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -28298,15 +28880,15 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: .LBB41_3: ; %end
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s35
-; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s34
-; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s31
-; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, s30
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: .LBB41_5: ; %end
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s31
+; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s30
+; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, vcc_hi
+; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, vcc_lo
; GFX9-NEXT: s_pack_ll_b32_b16 s18, s20, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s19, s21, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s20, s22, s93
@@ -28333,7 +28915,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s8, s8, s56
; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s47
; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s46
-; GFX9-NEXT: v_readlane_b32 s30, v30, 2
+; GFX9-NEXT: v_readlane_b32 s30, v30, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s16
@@ -28364,46 +28946,12 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_readlane_b32 s31, v30, 3
-; GFX9-NEXT: v_readlane_b32 s35, v30, 1
-; GFX9-NEXT: v_readlane_b32 s34, v30, 0
+; GFX9-NEXT: v_readlane_b32 s31, v30, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v15i64_to_v60i16_scalar:
; GFX11: ; %bb.0:
@@ -28423,7 +28971,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s94, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s40, s4, 16
; GFX11-NEXT: s_lshr_b32 s41, s5, 16
@@ -28455,9 +29003,46 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s94, -1
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -28518,7 +29103,7 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s93
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s92
@@ -28566,38 +29151,6 @@ define inreg <60 x i16> @bitcast_v15i64_to_v60i16_scalar(<15 x i64> inreg %a, i3
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -30000,10 +30553,8 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_writelane_b32 v30, s87, 29
; SI-NEXT: v_writelane_b32 v30, s96, 30
; SI-NEXT: v_writelane_b32 v30, s97, 31
-; SI-NEXT: v_writelane_b32 v30, s98, 32
-; SI-NEXT: v_writelane_b32 v30, s99, 33
-; SI-NEXT: v_writelane_b32 v30, s30, 34
-; SI-NEXT: v_writelane_b32 v30, s31, 35
+; SI-NEXT: v_writelane_b32 v30, s30, 32
+; SI-NEXT: v_writelane_b32 v30, s31, 33
; SI-NEXT: v_readfirstlane_b32 s7, v15
; SI-NEXT: v_readfirstlane_b32 s9, v14
; SI-NEXT: v_readfirstlane_b32 s11, v13
@@ -30015,25 +30566,25 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s79, v7
; SI-NEXT: v_readfirstlane_b32 s89, v6
; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s35, v3
-; SI-NEXT: v_readfirstlane_b32 s70, v2
-; SI-NEXT: v_readfirstlane_b32 s81, v1
-; SI-NEXT: v_readfirstlane_b32 s84, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s31, v3
+; SI-NEXT: v_readfirstlane_b32 s68, v2
+; SI-NEXT: v_readfirstlane_b32 s71, v1
+; SI-NEXT: v_readfirstlane_b32 s82, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s92, s28, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s68, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s82, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s80, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -30045,47 +30596,47 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s78, s79, 16
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
-; SI-NEXT: s_lshr_b32 s69, s70, 16
-; SI-NEXT: s_lshr_b32 s80, s81, 16
-; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
+; SI-NEXT: s_lshr_b32 s35, s68, 16
+; SI-NEXT: s_lshr_b32 s70, s71, 16
+; SI-NEXT: s_lshr_b32 s81, s82, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_4
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -30096,19 +30647,19 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -30144,61 +30695,70 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB43_3
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
@@ -30215,29 +30775,29 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_add_i32 s68, s68, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -30296,8 +30856,8 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s4, 0x30000
-; SI-NEXT: .LBB43_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v30, 34
+; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v30, 32
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -30328,9 +30888,7 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v27, s63
; SI-NEXT: v_mov_b32_e32 v28, s64
; SI-NEXT: v_mov_b32_e32 v29, s65
-; SI-NEXT: v_readlane_b32 s31, v30, 35
-; SI-NEXT: v_readlane_b32 s99, v30, 33
-; SI-NEXT: v_readlane_b32 s98, v30, 32
+; SI-NEXT: v_readlane_b32 s31, v30, 33
; SI-NEXT: v_readlane_b32 s97, v30, 31
; SI-NEXT: v_readlane_b32 s96, v30, 30
; SI-NEXT: v_readlane_b32 s87, v30, 29
@@ -30368,9 +30926,6 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB43_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB43_2
;
; VI-LABEL: bitcast_v60i16_to_v15i64_scalar:
; VI: ; %bb.0:
@@ -30411,312 +30966,399 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; VI-NEXT: v_writelane_b32 v30, s87, 29
; VI-NEXT: v_writelane_b32 v30, s30, 30
; VI-NEXT: v_writelane_b32 v30, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s11, v13
-; VI-NEXT: s_lshr_b32 s63, s11, 16
-; VI-NEXT: v_readfirstlane_b32 s13, v12
+; VI-NEXT: s_lshr_b32 s14, s20, 16
; VI-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 s62, s13, 16
-; VI-NEXT: v_readfirstlane_b32 s15, v11
-; VI-NEXT: v_writelane_b32 v31, s63, 0
-; VI-NEXT: s_lshr_b32 s61, s15, 16
-; VI-NEXT: v_readfirstlane_b32 s73, v10
-; VI-NEXT: v_writelane_b32 v31, s62, 1
-; VI-NEXT: v_readfirstlane_b32 s7, v15
-; VI-NEXT: v_readfirstlane_b32 s9, v14
-; VI-NEXT: s_lshr_b32 s60, s73, 16
-; VI-NEXT: v_readfirstlane_b32 s75, v9
+; VI-NEXT: s_lshr_b32 s42, s19, 16
+; VI-NEXT: v_writelane_b32 v31, s14, 0
+; VI-NEXT: s_lshr_b32 s43, s18, 16
+; VI-NEXT: v_readfirstlane_b32 s6, v15
+; VI-NEXT: v_readfirstlane_b32 s8, v14
+; VI-NEXT: v_readfirstlane_b32 s10, v13
+; VI-NEXT: v_readfirstlane_b32 s12, v12
+; VI-NEXT: v_readfirstlane_b32 s87, v11
+; VI-NEXT: v_readfirstlane_b32 s15, v10
+; VI-NEXT: v_readfirstlane_b32 s72, v9
; VI-NEXT: v_readfirstlane_b32 s77, v8
; VI-NEXT: v_readfirstlane_b32 s79, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s31, v5
-; VI-NEXT: v_readfirstlane_b32 s68, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: v_writelane_b32 v31, s61, 2
-; VI-NEXT: s_lshr_b32 s90, s29, 16
-; VI-NEXT: s_lshr_b32 s30, s28, 16
-; VI-NEXT: s_lshr_b32 s34, s27, 16
-; VI-NEXT: s_lshr_b32 s69, s26, 16
-; VI-NEXT: s_lshr_b32 s80, s25, 16
-; VI-NEXT: s_lshr_b32 s83, s24, 16
-; VI-NEXT: s_lshr_b32 s86, s23, 16
-; VI-NEXT: s_lshr_b32 s8, s22, 16
-; VI-NEXT: s_lshr_b32 s10, s21, 16
-; VI-NEXT: s_lshr_b32 s12, s20, 16
-; VI-NEXT: s_lshr_b32 s14, s19, 16
-; VI-NEXT: s_lshr_b32 s72, s18, 16
-; VI-NEXT: s_lshr_b32 s74, s17, 16
-; VI-NEXT: s_lshr_b32 s76, s16, 16
-; VI-NEXT: s_lshr_b32 s66, s7, 16
-; VI-NEXT: s_lshr_b32 s67, s9, 16
-; VI-NEXT: s_lshr_b32 s59, s75, 16
-; VI-NEXT: s_lshr_b32 s58, s77, 16
-; VI-NEXT: s_lshr_b32 s78, s79, 16
-; VI-NEXT: s_lshr_b32 s88, s89, 16
-; VI-NEXT: s_lshr_b32 s91, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
-; VI-NEXT: s_lshr_b32 s81, s82, 16
-; VI-NEXT: s_lshr_b32 s84, s85, 16
-; VI-NEXT: s_lshr_b32 s87, s6, 16
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; VI-NEXT: v_readfirstlane_b32 s34, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v31, s42, 1
+; VI-NEXT: s_lshr_b32 s35, s29, 16
+; VI-NEXT: s_lshr_b32 s68, s28, 16
+; VI-NEXT: s_lshr_b32 s70, s27, 16
+; VI-NEXT: s_lshr_b32 s81, s26, 16
+; VI-NEXT: s_lshr_b32 s84, s25, 16
+; VI-NEXT: s_lshr_b32 s7, s24, 16
+; VI-NEXT: s_lshr_b32 s11, s23, 16
+; VI-NEXT: s_lshr_b32 s73, s22, 16
+; VI-NEXT: s_lshr_b32 s75, s21, 16
+; VI-NEXT: s_lshr_b32 s41, s17, 16
+; VI-NEXT: s_lshr_b32 s5, s16, 16
+; VI-NEXT: s_lshr_b32 s66, s6, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
+; VI-NEXT: s_lshr_b32 s76, s10, 16
+; VI-NEXT: s_lshr_b32 s78, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s87, 16
+; VI-NEXT: s_lshr_b32 s90, s15, 16
+; VI-NEXT: s_lshr_b32 s91, s72, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s77, 16
+; VI-NEXT: s_lshr_b32 s30, s79, 16
+; VI-NEXT: s_lshr_b32 s31, s89, 16
+; VI-NEXT: s_lshr_b32 s67, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s71, s34, 16
+; VI-NEXT: s_lshr_b32 s82, s69, 16
+; VI-NEXT: s_lshr_b32 s85, s80, 16
+; VI-NEXT: s_lshr_b32 s9, s83, 16
+; VI-NEXT: s_lshr_b32 s13, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v31, s60, 3
+; VI-NEXT: v_writelane_b32 v31, s43, 2
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v31, s59, 4
-; VI-NEXT: v_writelane_b32 v31, s58, 5
-; VI-NEXT: s_cbranch_scc0 .LBB43_4
+; VI-NEXT: v_writelane_b32 v31, s41, 3
+; VI-NEXT: v_writelane_b32 v31, s5, 4
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s43, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s42, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s74, 16
+; VI-NEXT: s_lshl_b32 s41, s41, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s77
-; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s75
-; VI-NEXT: s_lshl_b32 s5, s59, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s72
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s73
-; VI-NEXT: s_lshl_b32 s5, s60, 16
-; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s15
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_or_b32 s60, s4, s5
+; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s13
-; VI-NEXT: s_lshl_b32 s5, s62, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s11
-; VI-NEXT: s_lshl_b32 s5, s63, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s10
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s64, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
+; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB43_3
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: v_writelane_b32 v31, s26, 5
+; VI-NEXT: s_mov_b32 s26, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s27
+; VI-NEXT: s_mov_b32 s27, s89
+; VI-NEXT: s_mov_b32 s89, s28
+; VI-NEXT: s_mov_b32 s28, s79
+; VI-NEXT: s_mov_b32 s79, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s77, s13
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: s_mov_b32 s10, s8
+; VI-NEXT: s_mov_b32 s9, s7
+; VI-NEXT: s_mov_b32 s8, s6
+; VI-NEXT: s_mov_b32 s7, s85
+; VI-NEXT: s_mov_b32 s85, s82
+; VI-NEXT: s_mov_b32 s82, s71
+; VI-NEXT: s_mov_b32 s71, s68
+; VI-NEXT: s_mov_b32 s68, s35
+; VI-NEXT: s_mov_b32 s35, s31
+; VI-NEXT: s_mov_b32 s31, s30
+; VI-NEXT: s_mov_b32 s30, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s91
+; VI-NEXT: s_mov_b32 s91, s90
+; VI-NEXT: s_mov_b32 s90, s88
+; VI-NEXT: s_mov_b32 s88, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s66
+; VI-NEXT: s_mov_b32 s6, s75
+; VI-NEXT: s_mov_b32 s75, s72
+; VI-NEXT: s_mov_b32 s72, s73
+; VI-NEXT: s_mov_b32 s73, s15
+; VI-NEXT: s_mov_b32 s15, s87
+; VI-NEXT: s_mov_b32 s87, s84
+; VI-NEXT: s_mov_b32 s84, s81
+; VI-NEXT: s_mov_b32 s81, s70
+; VI-NEXT: s_mov_b32 s70, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s70
+; VI-NEXT: s_mov_b32 s70, s81
+; VI-NEXT: s_mov_b32 s81, s84
+; VI-NEXT: s_mov_b32 s84, s87
+; VI-NEXT: s_mov_b32 s87, s15
+; VI-NEXT: s_mov_b32 s15, s73
+; VI-NEXT: s_mov_b32 s73, s72
+; VI-NEXT: s_mov_b32 s72, s75
+; VI-NEXT: s_mov_b32 s75, s6
+; VI-NEXT: s_mov_b32 s66, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s88
+; VI-NEXT: s_mov_b32 s88, s90
+; VI-NEXT: s_mov_b32 s90, s91
+; VI-NEXT: s_mov_b32 s91, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s30
+; VI-NEXT: s_mov_b32 s30, s31
+; VI-NEXT: s_mov_b32 s31, s35
+; VI-NEXT: s_mov_b32 s35, s68
+; VI-NEXT: s_mov_b32 s68, s71
+; VI-NEXT: s_mov_b32 s71, s82
+; VI-NEXT: s_mov_b32 s82, s85
+; VI-NEXT: s_mov_b32 s85, s7
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_mov_b32 s7, s9
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s11, s13
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s77
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s79
+; VI-NEXT: s_mov_b32 s79, s28
+; VI-NEXT: s_mov_b32 s28, s89
+; VI-NEXT: s_mov_b32 s89, s27
+; VI-NEXT: s_mov_b32 s27, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s26
+; VI-NEXT: v_readlane_b32 s26, v31, 5
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 4
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_add_i32 s17, s17, 3
+; VI-NEXT: v_readlane_b32 s14, v31, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s16, s74, 16
-; VI-NEXT: s_or_b32 s5, s16, s5
+; VI-NEXT: s_lshl_b32 s14, s14, 16
+; VI-NEXT: s_or_b32 s5, s14, s5
+; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_add_i32 s18, s18, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 2
; VI-NEXT: s_add_i32 s36, s4, 0x30000
-; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 1
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 0
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s28, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s48, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s29, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s6, s6, 3
+; VI-NEXT: s_add_i32 s86, s86, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, s86, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s85, s85, 3
+; VI-NEXT: s_add_i32 s83, s83, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s85, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, s83, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s82, s82, 3
+; VI-NEXT: s_add_i32 s80, s80, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s82, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, s80, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s89, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s79, s79, 3
; VI-NEXT: s_add_i32 s56, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s79, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s77, s77, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 5
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s75, s75, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 4
+; VI-NEXT: s_add_i32 s75, s72, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s75, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s73, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 3
+; VI-NEXT: s_add_i32 s73, s15, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s15, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 2
+; VI-NEXT: s_add_i32 s15, s87, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s13, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 1
+; VI-NEXT: s_add_i32 s13, s12, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s11, s11, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 0
+; VI-NEXT: s_add_i32 s11, s10, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s9, 3
+; VI-NEXT: s_add_i32 s9, s8, 3
; VI-NEXT: s_add_i32 s63, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s7, 3
+; VI-NEXT: s_add_i32 s7, s6, 3
; VI-NEXT: s_add_i32 s64, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s65, s4, 0x30000
-; VI-NEXT: .LBB43_3: ; %end
+; VI-NEXT: .LBB43_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v30, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -30785,13 +31427,6 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_4:
-; VI-NEXT: s_mov_b32 vcc_lo, s66
-; VI-NEXT: s_mov_b32 vcc_hi, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, vcc_hi
-; VI-NEXT: s_mov_b32 s66, vcc_lo
-; VI-NEXT: s_branch .LBB43_2
;
; GFX9-LABEL: bitcast_v60i16_to_v15i64_scalar:
; GFX9: ; %bb.0:
@@ -30891,10 +31526,18 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -30925,10 +31568,8 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v28, s64, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v29, s65, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -30961,7 +31602,7 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -31060,11 +31701,16 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB43_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -31096,8 +31742,6 @@ define inreg <15 x i64> @bitcast_v60i16_to_v15i64_scalar(<60 x i16> inreg %a, i3
; GFX11-NEXT: v_pk_add_u16 v28, s28, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v29, s29, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: s_branch .LBB43_2
; GFX11-NEXT: .LBB43_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -32100,7 +32744,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s34, s5, 16
; SI-NEXT: s_lshr_b32 s35, s7, 16
@@ -32131,9 +32775,47 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_u32 s4, s4, 3
; SI-NEXT: s_addc_u32 s5, s5, 0
; SI-NEXT: s_add_u32 s6, s6, 3
@@ -32193,9 +32875,9 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: .LBB45_3: ; %end
-; SI-NEXT: s_lshl_b32 s45, s30, 16
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: .LBB45_5: ; %end
+; SI-NEXT: s_lshl_b32 s45, vcc_lo, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s16, s16, s45
; SI-NEXT: s_and_b32 s17, s17, 0xffff
@@ -32337,38 +33019,6 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v15i64_to_v60f16_scalar:
; VI: ; %bb.0:
@@ -32380,10 +33030,8 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: v_writelane_b32 v30, s35, 1
; VI-NEXT: v_writelane_b32 v30, s36, 2
; VI-NEXT: v_writelane_b32 v30, s37, 3
-; VI-NEXT: v_writelane_b32 v30, s38, 4
-; VI-NEXT: v_writelane_b32 v30, s39, 5
-; VI-NEXT: v_writelane_b32 v30, s30, 6
-; VI-NEXT: v_writelane_b32 v30, s31, 7
+; VI-NEXT: v_writelane_b32 v30, s30, 4
+; VI-NEXT: v_writelane_b32 v30, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: v_readfirstlane_b32 s7, v14
@@ -32402,7 +33050,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s44, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s45, v0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s6, 16
; VI-NEXT: s_lshr_b32 s47, s7, 16
@@ -32426,16 +33074,54 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_u32 s7, s7, 3
; VI-NEXT: s_addc_u32 s6, s6, 0
; VI-NEXT: s_add_u32 s9, s9, 3
@@ -32488,38 +33174,38 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: s_lshr_b32 s89, s26, 16
; VI-NEXT: s_lshr_b32 s90, s25, 16
; VI-NEXT: s_lshr_b32 s91, s24, 16
-; VI-NEXT: s_lshr_b32 s30, s23, 16
-; VI-NEXT: s_lshr_b32 s31, s22, 16
-; VI-NEXT: s_lshr_b32 s34, s21, 16
-; VI-NEXT: s_lshr_b32 s35, s20, 16
-; VI-NEXT: s_lshr_b32 s36, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: s_lshr_b32 vcc_lo, s23, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s22, 16
+; VI-NEXT: s_lshr_b32 s30, s21, 16
+; VI-NEXT: s_lshr_b32 s31, s20, 16
+; VI-NEXT: s_lshr_b32 s34, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s39, 16
+; VI-NEXT: s_lshl_b32 s5, s37, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s38, 16
+; VI-NEXT: s_lshl_b32 s16, s36, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s37, 16
+; VI-NEXT: s_lshl_b32 s17, s35, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s36, 16
+; VI-NEXT: s_lshl_b32 s18, s34, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s19, s35, 16
+; VI-NEXT: s_lshl_b32 s19, s31, 16
; VI-NEXT: s_or_b32 s18, s18, s19
; VI-NEXT: s_and_b32 s19, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s20, s34, 16
+; VI-NEXT: s_lshl_b32 s20, s30, 16
; VI-NEXT: s_or_b32 s19, s19, s20
; VI-NEXT: s_and_b32 s20, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s21, s31, 16
+; VI-NEXT: s_lshl_b32 s21, vcc_hi, 16
; VI-NEXT: s_or_b32 s20, s20, s21
; VI-NEXT: s_and_b32 s21, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s22, s30, 16
+; VI-NEXT: s_lshl_b32 s22, vcc_lo, 16
; VI-NEXT: s_or_b32 s21, s21, s22
; VI-NEXT: s_and_b32 s22, 0xffff, s24
; VI-NEXT: s_lshl_b32 s23, s91, 16
@@ -32587,7 +33273,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: s_and_b32 s6, 0xffff, s6
; VI-NEXT: s_lshl_b32 s44, s46, 16
; VI-NEXT: s_or_b32 s6, s6, s44
-; VI-NEXT: v_readlane_b32 s30, v30, 6
+; VI-NEXT: v_readlane_b32 s30, v30, 4
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -32618,9 +33304,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v27, s8
; VI-NEXT: v_mov_b32_e32 v28, s7
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_readlane_b32 s31, v30, 7
-; VI-NEXT: v_readlane_b32 s39, v30, 5
-; VI-NEXT: v_readlane_b32 s38, v30, 4
+; VI-NEXT: v_readlane_b32 s31, v30, 5
; VI-NEXT: v_readlane_b32 s37, v30, 3
; VI-NEXT: v_readlane_b32 s36, v30, 2
; VI-NEXT: v_readlane_b32 s35, v30, 1
@@ -32630,38 +33314,6 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v15i64_to_v60f16_scalar:
; GFX9: ; %bb.0:
@@ -32669,10 +33321,8 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-NEXT: v_writelane_b32 v30, s34, 0
-; GFX9-NEXT: v_writelane_b32 v30, s35, 1
-; GFX9-NEXT: v_writelane_b32 v30, s30, 2
-; GFX9-NEXT: v_writelane_b32 v30, s31, 3
+; GFX9-NEXT: v_writelane_b32 v30, s30, 0
+; GFX9-NEXT: v_writelane_b32 v30, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s6, v15
; GFX9-NEXT: v_readfirstlane_b32 s7, v14
@@ -32691,7 +33341,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s44, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s45, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s6, 16
; GFX9-NEXT: s_lshr_b32 s47, s7, 16
@@ -32719,12 +33369,50 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_u32 s7, s7, 3
; GFX9-NEXT: s_addc_u32 s6, s6, 0
; GFX9-NEXT: s_add_u32 s9, s9, 3
@@ -32781,15 +33469,15 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s93, s22, 16
; GFX9-NEXT: s_lshr_b32 s94, s21, 16
; GFX9-NEXT: s_lshr_b32 s95, s20, 16
-; GFX9-NEXT: s_lshr_b32 s30, s19, 16
-; GFX9-NEXT: s_lshr_b32 s31, s18, 16
-; GFX9-NEXT: s_lshr_b32 s34, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: .LBB45_3: ; %end
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s35
-; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s34
-; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, s31
-; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, s30
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s19, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s17, 16
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: .LBB45_5: ; %end
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s16, s31
+; GFX9-NEXT: s_pack_ll_b32_b16 s5, s17, s30
+; GFX9-NEXT: s_pack_ll_b32_b16 s16, s18, vcc_hi
+; GFX9-NEXT: s_pack_ll_b32_b16 s17, s19, vcc_lo
; GFX9-NEXT: s_pack_ll_b32_b16 s18, s20, s95
; GFX9-NEXT: s_pack_ll_b32_b16 s19, s21, s94
; GFX9-NEXT: s_pack_ll_b32_b16 s20, s22, s93
@@ -32816,7 +33504,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s8, s8, s56
; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s47
; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s46
-; GFX9-NEXT: v_readlane_b32 s30, v30, 2
+; GFX9-NEXT: v_readlane_b32 s30, v30, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s16
@@ -32847,46 +33535,12 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v27, s8
; GFX9-NEXT: v_mov_b32_e32 v28, s7
; GFX9-NEXT: v_mov_b32_e32 v29, s6
-; GFX9-NEXT: v_readlane_b32 s31, v30, 3
-; GFX9-NEXT: v_readlane_b32 s35, v30, 1
-; GFX9-NEXT: v_readlane_b32 s34, v30, 0
+; GFX9-NEXT: v_readlane_b32 s31, v30, 1
; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v15i64_to_v60f16_scalar:
; GFX11: ; %bb.0:
@@ -32906,7 +33560,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX11-NEXT: v_readfirstlane_b32 s15, v0
; GFX11-NEXT: s_cmp_lg_u32 s40, 0
; GFX11-NEXT: s_mov_b32 s94, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s40, s4, 16
; GFX11-NEXT: s_lshr_b32 s41, s5, 16
@@ -32938,9 +33592,46 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s94, -1
+; GFX11-NEXT: ; implicit-def: $sgpr93
+; GFX11-NEXT: ; implicit-def: $sgpr92
+; GFX11-NEXT: ; implicit-def: $sgpr91
+; GFX11-NEXT: ; implicit-def: $sgpr90
+; GFX11-NEXT: ; implicit-def: $sgpr89
+; GFX11-NEXT: ; implicit-def: $sgpr88
+; GFX11-NEXT: ; implicit-def: $sgpr79
+; GFX11-NEXT: ; implicit-def: $sgpr78
+; GFX11-NEXT: ; implicit-def: $sgpr77
+; GFX11-NEXT: ; implicit-def: $sgpr76
+; GFX11-NEXT: ; implicit-def: $sgpr75
+; GFX11-NEXT: ; implicit-def: $sgpr74
+; GFX11-NEXT: ; implicit-def: $sgpr73
+; GFX11-NEXT: ; implicit-def: $sgpr72
+; GFX11-NEXT: ; implicit-def: $sgpr63
+; GFX11-NEXT: ; implicit-def: $sgpr62
+; GFX11-NEXT: ; implicit-def: $sgpr61
+; GFX11-NEXT: ; implicit-def: $sgpr60
+; GFX11-NEXT: ; implicit-def: $sgpr59
+; GFX11-NEXT: ; implicit-def: $sgpr58
+; GFX11-NEXT: ; implicit-def: $sgpr57
+; GFX11-NEXT: ; implicit-def: $sgpr56
+; GFX11-NEXT: ; implicit-def: $sgpr47
+; GFX11-NEXT: ; implicit-def: $sgpr46
+; GFX11-NEXT: ; implicit-def: $sgpr45
+; GFX11-NEXT: ; implicit-def: $sgpr44
+; GFX11-NEXT: ; implicit-def: $sgpr43
+; GFX11-NEXT: ; implicit-def: $sgpr42
+; GFX11-NEXT: ; implicit-def: $sgpr41
+; GFX11-NEXT: ; implicit-def: $sgpr40
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_u32 s5, s5, 3
; GFX11-NEXT: s_addc_u32 s4, s4, 0
; GFX11-NEXT: s_add_u32 s7, s7, 3
@@ -33001,7 +33692,7 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX11-NEXT: s_lshr_b32 s91, s2, 16
; GFX11-NEXT: s_lshr_b32 s92, s1, 16
; GFX11-NEXT: s_lshr_b32 s93, s0, 16
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s93
; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s92
@@ -33049,38 +33740,6 @@ define inreg <60 x half> @bitcast_v15i64_to_v60f16_scalar(<15 x i64> inreg %a, i
; GFX11-NEXT: v_dual_mov_b32 v26, s7 :: v_dual_mov_b32 v27, s6
; GFX11-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v29, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $sgpr93
-; GFX11-NEXT: ; implicit-def: $sgpr92
-; GFX11-NEXT: ; implicit-def: $sgpr91
-; GFX11-NEXT: ; implicit-def: $sgpr90
-; GFX11-NEXT: ; implicit-def: $sgpr89
-; GFX11-NEXT: ; implicit-def: $sgpr88
-; GFX11-NEXT: ; implicit-def: $sgpr79
-; GFX11-NEXT: ; implicit-def: $sgpr78
-; GFX11-NEXT: ; implicit-def: $sgpr77
-; GFX11-NEXT: ; implicit-def: $sgpr76
-; GFX11-NEXT: ; implicit-def: $sgpr75
-; GFX11-NEXT: ; implicit-def: $sgpr74
-; GFX11-NEXT: ; implicit-def: $sgpr73
-; GFX11-NEXT: ; implicit-def: $sgpr72
-; GFX11-NEXT: ; implicit-def: $sgpr63
-; GFX11-NEXT: ; implicit-def: $sgpr62
-; GFX11-NEXT: ; implicit-def: $sgpr61
-; GFX11-NEXT: ; implicit-def: $sgpr60
-; GFX11-NEXT: ; implicit-def: $sgpr59
-; GFX11-NEXT: ; implicit-def: $sgpr58
-; GFX11-NEXT: ; implicit-def: $sgpr57
-; GFX11-NEXT: ; implicit-def: $sgpr56
-; GFX11-NEXT: ; implicit-def: $sgpr47
-; GFX11-NEXT: ; implicit-def: $sgpr46
-; GFX11-NEXT: ; implicit-def: $sgpr45
-; GFX11-NEXT: ; implicit-def: $sgpr44
-; GFX11-NEXT: ; implicit-def: $sgpr43
-; GFX11-NEXT: ; implicit-def: $sgpr42
-; GFX11-NEXT: ; implicit-def: $sgpr41
-; GFX11-NEXT: ; implicit-def: $sgpr40
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -34627,10 +35286,8 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_writelane_b32 v32, s87, 29
; SI-NEXT: v_writelane_b32 v32, s96, 30
; SI-NEXT: v_writelane_b32 v32, s97, 31
-; SI-NEXT: v_writelane_b32 v32, s98, 32
-; SI-NEXT: v_writelane_b32 v32, s99, 33
-; SI-NEXT: v_writelane_b32 v32, s30, 34
-; SI-NEXT: v_writelane_b32 v32, s31, 35
+; SI-NEXT: v_writelane_b32 v32, s30, 32
+; SI-NEXT: v_writelane_b32 v32, s31, 33
; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: v_readfirstlane_b32 s8, v14
; SI-NEXT: v_readfirstlane_b32 s10, v13
@@ -34644,23 +35301,23 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s90, v5
; SI-NEXT: v_readfirstlane_b32 s92, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s30, v2
-; SI-NEXT: v_readfirstlane_b32 s35, v1
-; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: s_lshr_b32 s31, s29, 16
-; SI-NEXT: s_lshr_b32 s68, s28, 16
-; SI-NEXT: s_lshr_b32 s71, s27, 16
-; SI-NEXT: s_lshr_b32 s81, s26, 16
-; SI-NEXT: s_lshr_b32 s82, s25, 16
-; SI-NEXT: s_lshr_b32 s83, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v0
+; SI-NEXT: s_lshr_b32 vcc_hi, s29, 16
+; SI-NEXT: s_lshr_b32 s34, s28, 16
+; SI-NEXT: s_lshr_b32 s69, s27, 16
+; SI-NEXT: s_lshr_b32 s71, s26, 16
+; SI-NEXT: s_lshr_b32 s80, s25, 16
+; SI-NEXT: s_lshr_b32 s81, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -34674,63 +35331,63 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s34, s30, 16
-; SI-NEXT: s_lshr_b32 s69, s35, 16
-; SI-NEXT: s_lshr_b32 s80, s70, 16
+; SI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s35, s31, 16
+; SI-NEXT: s_lshr_b32 s70, s68, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_3
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -34771,11 +35428,20 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB47_4
-; SI-NEXT: .LBB47_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s99
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s97
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s96
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -34785,7 +35451,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s87
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -34795,20 +35461,20 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s86
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s84
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -34818,7 +35484,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s83
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -34827,11 +35493,11 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s82
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s81
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
@@ -34842,7 +35508,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s80
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
@@ -34851,11 +35517,11 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s71
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
@@ -34866,7 +35532,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s34
; SI-NEXT: v_cvt_f32_f16_e32 v14, s28
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
@@ -34875,11 +35541,11 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_or_b32_e32 v12, v14, v12
; SI-NEXT: v_cvt_f32_f16_e32 v14, s29
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s70
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -34887,11 +35553,11 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s68
; SI-NEXT: v_or_b32_e32 v13, v14, v13
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s69
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -34899,10 +35565,10 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s30
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v17, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v18, s95
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
@@ -35013,11 +35679,8 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v28, v29, v28
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v30
; SI-NEXT: v_or_b32_e32 v29, v31, v29
-; SI-NEXT: s_branch .LBB47_5
-; SI-NEXT: .LBB47_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB47_2
-; SI-NEXT: .LBB47_4:
+; SI-NEXT: s_branch .LBB47_6
+; SI-NEXT: .LBB47_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -35050,11 +35713,9 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB47_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s31, v32, 35
-; SI-NEXT: v_readlane_b32 s99, v32, 33
-; SI-NEXT: v_readlane_b32 s98, v32, 32
+; SI-NEXT: .LBB47_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 32
+; SI-NEXT: v_readlane_b32 s31, v32, 33
; SI-NEXT: v_readlane_b32 s97, v32, 31
; SI-NEXT: v_readlane_b32 s96, v32, 30
; SI-NEXT: v_readlane_b32 s87, v32, 29
@@ -35132,136 +35793,136 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s6, v15
-; VI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; VI-NEXT: v_readfirstlane_b32 s8, v14
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 vcc_hi, s8, 16
+; VI-NEXT: s_lshr_b32 s75, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v13
-; VI-NEXT: v_writelane_b32 v33, vcc_lo, 0
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_lshr_b32 s63, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v12
-; VI-NEXT: v_writelane_b32 v33, vcc_hi, 1
+; VI-NEXT: v_writelane_b32 v33, s75, 0
; VI-NEXT: s_lshr_b32 s62, s12, 16
; VI-NEXT: v_readfirstlane_b32 s14, v11
-; VI-NEXT: v_writelane_b32 v33, s63, 2
+; VI-NEXT: v_writelane_b32 v33, s63, 1
+; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: s_lshr_b32 s61, s14, 16
; VI-NEXT: v_readfirstlane_b32 s72, v10
-; VI-NEXT: v_writelane_b32 v33, s62, 3
-; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: v_readfirstlane_b32 s74, v9
; VI-NEXT: v_readfirstlane_b32 s76, v8
; VI-NEXT: v_readfirstlane_b32 s78, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s30, v5
-; VI-NEXT: v_readfirstlane_b32 s35, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: v_writelane_b32 v33, s61, 4
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v5
+; VI-NEXT: v_readfirstlane_b32 s31, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v33, s62, 2
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s88, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s68, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s81, s24, 16
-; VI-NEXT: s_lshr_b32 s84, s23, 16
-; VI-NEXT: s_lshr_b32 s86, s22, 16
-; VI-NEXT: s_lshr_b32 s9, s21, 16
-; VI-NEXT: s_lshr_b32 s13, s20, 16
-; VI-NEXT: s_lshr_b32 s15, s19, 16
-; VI-NEXT: s_lshr_b32 s73, s18, 16
-; VI-NEXT: s_lshr_b32 s75, s17, 16
-; VI-NEXT: s_lshr_b32 s77, s16, 16
+; VI-NEXT: s_lshr_b32 s77, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s68, s25, 16
+; VI-NEXT: s_lshr_b32 s71, s24, 16
+; VI-NEXT: s_lshr_b32 s82, s23, 16
+; VI-NEXT: s_lshr_b32 s84, s22, 16
+; VI-NEXT: s_lshr_b32 s87, s21, 16
+; VI-NEXT: s_lshr_b32 s9, s20, 16
+; VI-NEXT: s_lshr_b32 s11, s19, 16
+; VI-NEXT: s_lshr_b32 s13, s18, 16
+; VI-NEXT: s_lshr_b32 s15, s17, 16
+; VI-NEXT: s_lshr_b32 s73, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: s_lshr_b32 s59, s74, 16
; VI-NEXT: s_lshr_b32 s58, s76, 16
; VI-NEXT: s_lshr_b32 s57, s78, 16
-; VI-NEXT: s_lshr_b32 s64, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
-; VI-NEXT: s_lshr_b32 s83, s82, 16
-; VI-NEXT: s_lshr_b32 s87, s85, 16
-; VI-NEXT: s_lshr_b32 s11, s7, 16
+; VI-NEXT: s_lshr_b32 s88, s89, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
+; VI-NEXT: s_lshr_b32 s81, s80, 16
+; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 s7, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v33, s60, 5
+; VI-NEXT: v_writelane_b32 v33, s61, 3
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s59, 6
-; VI-NEXT: v_writelane_b32 v33, s58, 7
-; VI-NEXT: s_cbranch_scc0 .LBB47_3
+; VI-NEXT: v_writelane_b32 v33, s60, 4
+; VI-NEXT: v_writelane_b32 v33, s59, 5
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s77, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s73, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s75, 16
+; VI-NEXT: s_lshl_b32 s41, s15, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s77, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s64, 16
-; VI-NEXT: s_mov_b32 s91, s56
+; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_mov_b32 s90, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s78
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s88, s57
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s77, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s88, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s59, 16
@@ -35279,152 +35940,158 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s63, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
-; VI-NEXT: s_mov_b32 s90, s64
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s64, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_mov_b32 s75, s65
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB47_4
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s90, s56
+; VI-NEXT: s_mov_b32 s77, s57
+; VI-NEXT: s_mov_b32 s88, s58
+; VI-NEXT: s_mov_b32 s75, s65
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v29, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s77
-; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_mov_b32_e32 v0, s73
+; VI-NEXT: v_mov_b32_e32 v2, s15
; VI-NEXT: v_add_f16_sdwa v0, v0, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v29
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v29
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s73
+; VI-NEXT: v_mov_b32_e32 v2, s13
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v29
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_add_f16_sdwa v3, v3, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v29
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s13
+; VI-NEXT: v_mov_b32_e32 v4, s9
; VI-NEXT: v_add_f16_sdwa v4, v4, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v29
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v5, s87
; VI-NEXT: v_add_f16_sdwa v5, v5, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v29
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s86
+; VI-NEXT: v_mov_b32_e32 v6, s84
; VI-NEXT: v_add_f16_sdwa v6, v6, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v29
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s84
+; VI-NEXT: v_mov_b32_e32 v7, s82
; VI-NEXT: v_add_f16_sdwa v7, v7, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v29
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s81
+; VI-NEXT: v_mov_b32_e32 v8, s71
; VI-NEXT: v_add_f16_sdwa v8, v8, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v29
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s70
+; VI-NEXT: v_mov_b32_e32 v9, s68
; VI-NEXT: v_add_f16_sdwa v9, v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v29
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s68
+; VI-NEXT: v_mov_b32_e32 v10, s34
; VI-NEXT: v_add_f16_sdwa v10, v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v29
; VI-NEXT: v_or_b32_e32 v10, v11, v10
-; VI-NEXT: v_mov_b32_e32 v11, s31
+; VI-NEXT: v_mov_b32_e32 v11, vcc_hi
; VI-NEXT: v_add_f16_sdwa v11, v11, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v29
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s79
+; VI-NEXT: v_mov_b32_e32 v12, s91
; VI-NEXT: v_add_f16_sdwa v12, v12, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v29
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s91
+; VI-NEXT: v_mov_b32_e32 v13, s90
; VI-NEXT: v_add_f16_sdwa v13, v13, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v29
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s11
+; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_add_f16_sdwa v14, v14, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s7, v29
+; VI-NEXT: v_add_f16_e32 v15, s86, v29
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s87
+; VI-NEXT: v_mov_b32_e32 v15, s85
; VI-NEXT: v_add_f16_sdwa v15, v15, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s85, v29
+; VI-NEXT: v_add_f16_e32 v16, s83, v29
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s83
+; VI-NEXT: v_mov_b32_e32 v16, s81
; VI-NEXT: v_add_f16_sdwa v16, v16, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s82, v29
+; VI-NEXT: v_add_f16_e32 v17, s80, v29
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s80
+; VI-NEXT: v_mov_b32_e32 v17, s70
; VI-NEXT: v_add_f16_sdwa v17, v17, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s71, v29
+; VI-NEXT: v_add_f16_e32 v18, s69, v29
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s69
+; VI-NEXT: v_mov_b32_e32 v18, s35
; VI-NEXT: v_add_f16_sdwa v18, v18, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v19, s35, v29
+; VI-NEXT: v_add_f16_e32 v19, s31, v29
; VI-NEXT: v_or_b32_e32 v18, v19, v18
-; VI-NEXT: v_mov_b32_e32 v19, s34
+; VI-NEXT: v_mov_b32_e32 v19, s30
; VI-NEXT: v_add_f16_sdwa v19, v19, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v20, s30, v29
+; VI-NEXT: v_add_f16_e32 v20, vcc_lo, v29
; VI-NEXT: v_or_b32_e32 v19, v20, v19
-; VI-NEXT: v_mov_b32_e32 v20, s90
+; VI-NEXT: v_mov_b32_e32 v20, s79
; VI-NEXT: v_add_f16_sdwa v20, v20, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s89, v29
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s88
+; VI-NEXT: v_mov_b32_e32 v21, s77
; VI-NEXT: v_add_f16_sdwa v21, v21, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s78, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 7
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s4
+; VI-NEXT: v_mov_b32_e32 v22, s88
; VI-NEXT: v_add_f16_sdwa v22, v22, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s76, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 6
+; VI-NEXT: v_readlane_b32 s4, v33, 5
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s74, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 5
+; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s72, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 4
+; VI-NEXT: v_readlane_b32 s4, v33, 3
; VI-NEXT: v_or_b32_e32 v24, v25, v24
; VI-NEXT: v_mov_b32_e32 v25, s4
; VI-NEXT: v_add_f16_sdwa v25, v25, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s14, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 2
; VI-NEXT: v_or_b32_e32 v25, v26, v25
; VI-NEXT: v_mov_b32_e32 v26, s4
; VI-NEXT: v_add_f16_sdwa v26, v26, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s12, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v26, v27, v26
; VI-NEXT: v_mov_b32_e32 v27, s4
; VI-NEXT: v_add_f16_sdwa v27, v27, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s10, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 1
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v27, v28, v27
; VI-NEXT: v_mov_b32_e32 v28, s4
; VI-NEXT: v_add_f16_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v30, s8, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v28, v30, v28
-; VI-NEXT: v_mov_b32_e32 v30, s4
+; VI-NEXT: v_mov_b32_e32 v30, s75
; VI-NEXT: v_add_f16_sdwa v30, v30, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v29, s6, v29
; VI-NEXT: v_or_b32_e32 v29, v29, v30
-; VI-NEXT: s_branch .LBB47_5
-; VI-NEXT: .LBB47_3:
-; VI-NEXT: s_mov_b32 s90, s64
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s91, s56
-; VI-NEXT: s_mov_b32 s88, s57
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB47_2
-; VI-NEXT: .LBB47_4:
+; VI-NEXT: s_branch .LBB47_6
+; VI-NEXT: .LBB47_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -35457,7 +36124,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB47_5: ; %end
+; VI-NEXT: .LBB47_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -35595,10 +36262,18 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v29, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v29 op_sel_hi:[1,0]
@@ -35630,10 +36305,8 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_pk_add_f16 v27, s63, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v28, s64, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v29, s65, v29 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -35666,7 +36339,7 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -35765,11 +36438,16 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB47_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -35801,8 +36479,6 @@ define inreg <15 x i64> @bitcast_v60f16_to_v15i64_scalar(<60 x half> inreg %a, i
; GFX11-NEXT: v_pk_add_f16 v28, 0x200, s28 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v29, 0x200, s29 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: s_branch .LBB47_2
; GFX11-NEXT: .LBB47_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -36727,7 +37403,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB49_3
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s64, s15, 16
; SI-NEXT: s_lshr_b32 s55, s41, 16
@@ -36758,64 +37434,12 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_4
-; SI-NEXT: .LBB49_2: ; %cmp.true
-; SI-NEXT: v_add_f64 v[28:29], s[14:15], 1.0
-; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
-; SI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
-; SI-NEXT: v_lshr_b64 v[30:31], v[28:29], 16
-; SI-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
-; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
-; SI-NEXT: v_lshr_b64 v[31:32], v[26:27], 16
-; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
-; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
-; SI-NEXT: v_lshr_b64 v[32:33], v[24:25], 16
-; SI-NEXT: v_add_f64 v[18:19], s[6:7], 1.0
-; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
-; SI-NEXT: v_lshr_b64 v[33:34], v[22:23], 16
-; SI-NEXT: v_lshr_b64 v[48:49], v[10:11], 16
-; SI-NEXT: v_add_f64 v[16:17], s[4:5], 1.0
-; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
-; SI-NEXT: v_lshr_b64 v[34:35], v[20:21], 16
-; SI-NEXT: v_lshr_b64 v[49:50], v[8:9], 16
-; SI-NEXT: v_add_f64 v[14:15], s[42:43], 1.0
-; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
-; SI-NEXT: v_lshr_b64 v[35:36], v[18:19], 16
-; SI-NEXT: v_lshr_b64 v[50:51], v[6:7], 16
-; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
-; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
-; SI-NEXT: v_lshr_b64 v[36:37], v[16:17], 16
-; SI-NEXT: v_lshr_b64 v[51:52], v[4:5], 16
-; SI-NEXT: v_lshr_b64 v[37:38], v[14:15], 16
-; SI-NEXT: v_lshr_b64 v[52:53], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[38:39], v[12:13], 16
-; SI-NEXT: v_lshr_b64 v[53:54], v[0:1], 16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v23
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v15
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v13
-; SI-NEXT: s_waitcnt expcnt(5)
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v11
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v9
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v7
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v5
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: s_branch .LBB49_5
-; SI-NEXT: .LBB49_3:
-; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr34
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr35
@@ -36845,8 +37469,66 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; SI-NEXT: ; implicit-def: $sgpr55
; SI-NEXT: ; implicit-def: $sgpr64
; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB49_2
-; SI-NEXT: .LBB49_4:
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f64 v[28:29], s[14:15], 1.0
+; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
+; SI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
+; SI-NEXT: v_lshr_b64 v[30:31], v[28:29], 16
+; SI-NEXT: v_add_f64 v[22:23], s[10:11], 1.0
+; SI-NEXT: v_add_f64 v[10:11], s[26:27], 1.0
+; SI-NEXT: v_lshr_b64 v[31:32], v[26:27], 16
+; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
+; SI-NEXT: v_add_f64 v[8:9], s[24:25], 1.0
+; SI-NEXT: v_lshr_b64 v[32:33], v[24:25], 16
+; SI-NEXT: v_add_f64 v[18:19], s[6:7], 1.0
+; SI-NEXT: v_add_f64 v[6:7], s[22:23], 1.0
+; SI-NEXT: v_lshr_b64 v[33:34], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[48:49], v[10:11], 16
+; SI-NEXT: v_add_f64 v[16:17], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[4:5], s[20:21], 1.0
+; SI-NEXT: v_lshr_b64 v[34:35], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[8:9], 16
+; SI-NEXT: v_add_f64 v[14:15], s[42:43], 1.0
+; SI-NEXT: v_add_f64 v[2:3], s[18:19], 1.0
+; SI-NEXT: v_lshr_b64 v[35:36], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[50:51], v[6:7], 16
+; SI-NEXT: v_add_f64 v[12:13], s[28:29], 1.0
+; SI-NEXT: v_add_f64 v[0:1], s[16:17], 1.0
+; SI-NEXT: v_lshr_b64 v[36:37], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[51:52], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[37:38], v[14:15], 16
+; SI-NEXT: v_lshr_b64 v[52:53], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[38:39], v[12:13], 16
+; SI-NEXT: v_lshr_b64 v[53:54], v[0:1], 16
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v19
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v15
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v13
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v11
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v9
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v7
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v5
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
+; SI-NEXT: s_branch .LBB49_6
+; SI-NEXT: .LBB49_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -36899,7 +37581,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v60, s35
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v61, s34
-; SI-NEXT: v_mov_b32_e32 v53, s30
+; SI-NEXT: v_mov_b32_e32 v53, vcc_lo
; SI-NEXT: v_mov_b32_e32 v52, s94
; SI-NEXT: v_mov_b32_e32 v51, s92
; SI-NEXT: v_mov_b32_e32 v50, s90
@@ -36914,7 +37596,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v32, s56
; SI-NEXT: v_mov_b32_e32 v31, s44
; SI-NEXT: v_mov_b32_e32 v30, s46
-; SI-NEXT: .LBB49_5: ; %end
+; SI-NEXT: .LBB49_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -37064,10 +37746,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_writelane_b32 v60, s35, 1
; VI-NEXT: v_writelane_b32 v60, s36, 2
; VI-NEXT: v_writelane_b32 v60, s37, 3
-; VI-NEXT: v_writelane_b32 v60, s38, 4
-; VI-NEXT: v_writelane_b32 v60, s39, 5
-; VI-NEXT: v_writelane_b32 v60, s30, 6
-; VI-NEXT: v_writelane_b32 v60, s31, 7
+; VI-NEXT: v_writelane_b32 v60, s30, 4
+; VI-NEXT: v_writelane_b32 v60, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s9, v15
; VI-NEXT: v_readfirstlane_b32 s8, v14
@@ -37086,7 +37766,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s9, 16
; VI-NEXT: s_lshr_b32 s77, s8, 16
@@ -37103,23 +37783,61 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: s_lshr_b32 s60, s7, 16
; VI-NEXT: s_lshr_b32 s91, s6, 16
; VI-NEXT: s_lshr_b32 s61, s5, 16
-; VI-NEXT: s_lshr_b32 s30, s4, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s4, 16
; VI-NEXT: s_lshr_b32 s62, s29, 16
-; VI-NEXT: s_lshr_b32 s31, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s28, 16
; VI-NEXT: s_lshr_b32 s63, s27, 16
-; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s30, s26, 16
; VI-NEXT: s_lshr_b32 s72, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
; VI-NEXT: s_lshr_b32 s73, s23, 16
-; VI-NEXT: s_lshr_b32 s36, s22, 16
+; VI-NEXT: s_lshr_b32 s34, s22, 16
; VI-NEXT: s_lshr_b32 s74, s21, 16
-; VI-NEXT: s_lshr_b32 s37, s20, 16
+; VI-NEXT: s_lshr_b32 s35, s20, 16
; VI-NEXT: s_lshr_b32 s75, s19, 16
-; VI-NEXT: s_lshr_b32 s38, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s18, 16
; VI-NEXT: s_lshr_b32 s76, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[44:45], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[44:45], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; VI-NEXT: s_cselect_b32 s44, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s44, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[28:29], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
@@ -37165,40 +37883,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_lshrrev_b32_e32 v46, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v59, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v56, 16, v0
-; VI-NEXT: s_branch .LBB49_5
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: s_branch .LBB49_6
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -37229,14 +37915,14 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v27, s11
; VI-NEXT: v_mov_b32_e32 v28, s8
; VI-NEXT: v_mov_b32_e32 v29, s9
-; VI-NEXT: v_mov_b32_e32 v56, s39
-; VI-NEXT: v_mov_b32_e32 v46, s38
-; VI-NEXT: v_mov_b32_e32 v44, s37
-; VI-NEXT: v_mov_b32_e32 v42, s36
-; VI-NEXT: v_mov_b32_e32 v40, s35
-; VI-NEXT: v_mov_b32_e32 v54, s34
-; VI-NEXT: v_mov_b32_e32 v52, s31
-; VI-NEXT: v_mov_b32_e32 v50, s30
+; VI-NEXT: v_mov_b32_e32 v56, s37
+; VI-NEXT: v_mov_b32_e32 v46, s36
+; VI-NEXT: v_mov_b32_e32 v44, s35
+; VI-NEXT: v_mov_b32_e32 v42, s34
+; VI-NEXT: v_mov_b32_e32 v40, s31
+; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: v_mov_b32_e32 v52, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v50, vcc_lo
; VI-NEXT: v_mov_b32_e32 v48, s91
; VI-NEXT: v_mov_b32_e32 v38, s90
; VI-NEXT: v_mov_b32_e32 v36, s89
@@ -37259,7 +37945,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_mov_b32_e32 v37, s56
; VI-NEXT: v_mov_b32_e32 v35, s47
; VI-NEXT: v_mov_b32_e32 v33, s46
-; VI-NEXT: .LBB49_5: ; %end
+; VI-NEXT: .LBB49_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v56, 16, v56
; VI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
; VI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
@@ -37322,7 +38008,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v35
; VI-NEXT: v_or_b32_sdwa v28, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v33
-; VI-NEXT: v_readlane_b32 s30, v60, 6
+; VI-NEXT: v_readlane_b32 s30, v60, 4
; VI-NEXT: v_or_b32_sdwa v11, v11, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v13, v13, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v15, v15, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -37333,9 +38019,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; VI-NEXT: v_or_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v60, 7
-; VI-NEXT: v_readlane_b32 s39, v60, 5
-; VI-NEXT: v_readlane_b32 s38, v60, 4
+; VI-NEXT: v_readlane_b32 s31, v60, 5
; VI-NEXT: v_readlane_b32 s37, v60, 3
; VI-NEXT: v_readlane_b32 s36, v60, 2
; VI-NEXT: v_readlane_b32 s35, v60, 1
@@ -37364,10 +38048,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s9, v15
; GFX9-NEXT: v_readfirstlane_b32 s8, v14
@@ -37386,7 +38068,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s9, 16
; GFX9-NEXT: s_lshr_b32 s77, s8, 16
@@ -37411,15 +38093,53 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: s_lshr_b32 s72, s25, 16
; GFX9-NEXT: s_lshr_b32 s95, s24, 16
; GFX9-NEXT: s_lshr_b32 s73, s23, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 16
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 16
; GFX9-NEXT: s_lshr_b32 s74, s21, 16
-; GFX9-NEXT: s_lshr_b32 s31, s20, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s20, 16
; GFX9-NEXT: s_lshr_b32 s75, s19, 16
-; GFX9-NEXT: s_lshr_b32 s34, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s18, 16
; GFX9-NEXT: s_lshr_b32 s76, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[44:45], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[44:45], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; GFX9-NEXT: s_cselect_b32 s44, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s44, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[28:29], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
@@ -37465,40 +38185,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v0
-; GFX9-NEXT: s_branch .LBB49_5
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: s_branch .LBB49_6
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -37529,10 +38217,10 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v27, s11
; GFX9-NEXT: v_mov_b32_e32 v28, s8
; GFX9-NEXT: v_mov_b32_e32 v29, s9
-; GFX9-NEXT: v_mov_b32_e32 v56, s35
-; GFX9-NEXT: v_mov_b32_e32 v46, s34
-; GFX9-NEXT: v_mov_b32_e32 v44, s31
-; GFX9-NEXT: v_mov_b32_e32 v42, s30
+; GFX9-NEXT: v_mov_b32_e32 v56, s31
+; GFX9-NEXT: v_mov_b32_e32 v46, s30
+; GFX9-NEXT: v_mov_b32_e32 v44, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v42, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v40, s95
; GFX9-NEXT: v_mov_b32_e32 v54, s94
; GFX9-NEXT: v_mov_b32_e32 v52, s93
@@ -37559,7 +38247,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v37, s56
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
-; GFX9-NEXT: .LBB49_5: ; %end
+; GFX9-NEXT: .LBB49_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -37614,7 +38302,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
@@ -37633,9 +38321,7 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -37659,12 +38345,12 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s15, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s15, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s72, s14, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s11, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s11, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s10, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s8, 16
@@ -37692,9 +38378,46 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-TRUE16-NEXT: s_lshr_b32 s93, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s63, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB49_3
+; GFX11-TRUE16-NEXT: .LBB49_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[28:29], s[14:15], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
@@ -37740,40 +38463,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB49_5
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
-; GFX11-TRUE16-NEXT: .LBB49_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -37802,9 +38493,9 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s76 :: v_dual_mov_b32 v49, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s75 :: v_dual_mov_b32 v39, s44
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s74 :: v_dual_mov_b32 v37, s43
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s72 :: v_dual_mov_b32 v35, s42
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v82, s62 :: v_dual_mov_b32 v33, s41
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s72 :: v_dual_mov_b32 v35, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v82, s62 :: v_dual_mov_b32 v33, s40
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v80.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v83.l
@@ -37856,12 +38547,12 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s15, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s15, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s72, s14, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s11, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s11, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s10, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s8, 16
@@ -37889,56 +38580,9 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-FAKE16-NEXT: s_lshr_b32 s93, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s63, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[14:15], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[10:11], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[29:30], s[8:9], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[20:21], s[6:7], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[22:23], s[4:5], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[12:13], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[28:29], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[31:32], s[26:27], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[10:11], s[24:25], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[12:13], s[22:23], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[5:6], s[20:21], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[7:8], s[18:19], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[33:34], s[16:17], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[0:1], s[2:3], 1.0
-; GFX11-FAKE16-NEXT: v_add_f64 v[2:3], s[0:1], 1.0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v26
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v25
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v28
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v27
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v48, 16, v30
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v24, 16, v29
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v21
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v20
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v23
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v22
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v16
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v15
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v18
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v17
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v64, 16, v32
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v31
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v11
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v10
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v13
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v12
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v70, 16, v8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v68, 16, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v80, 16, v34
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v33
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v82, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v83, 16, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB49_5
-; GFX11-FAKE16-NEXT: .LBB49_3:
+; GFX11-FAKE16-NEXT: s_branch .LBB49_3
+; GFX11-FAKE16-NEXT: .LBB49_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
@@ -37966,11 +38610,63 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
-; GFX11-FAKE16-NEXT: .LBB49_4:
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: .LBB49_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[14:15], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[10:11], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[29:30], s[8:9], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[20:21], s[6:7], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[22:23], s[4:5], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[15:16], s[12:13], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[17:18], s[28:29], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[31:32], s[26:27], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[10:11], s[24:25], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[12:13], s[22:23], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[5:6], s[20:21], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[7:8], s[18:19], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[33:34], s[16:17], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[0:1], s[2:3], 1.0
+; GFX11-FAKE16-NEXT: v_add_f64 v[2:3], s[0:1], 1.0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v37, 16, v26
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v35, 16, v25
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v38, 16, v28
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v36, 16, v27
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v48, 16, v30
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v24, 16, v29
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v50, 16, v21
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v39, 16, v20
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v51, 16, v23
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v49, 16, v22
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v53, 16, v16
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v15
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v54, 16, v18
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v52, 16, v17
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v64, 16, v32
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v31
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v66, 16, v11
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v55, 16, v10
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v67, 16, v13
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v65, 16, v12
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v69, 16, v6
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v70, 16, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v68, 16, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v80, 16, v34
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v33
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v82, 16, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v83, 16, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
+; GFX11-FAKE16-NEXT: s_branch .LBB49_6
+; GFX11-FAKE16-NEXT: .LBB49_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v33, s16
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s20 :: v_dual_mov_b32 v12, s22
@@ -38000,8 +38696,8 @@ define inreg <60 x i16> @bitcast_v15f64_to_v60i16_scalar(<15 x double> inreg %a,
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v67, s58 :: v_dual_mov_b32 v54, s47
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, s46 :: v_dual_mov_b32 v50, s44
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s45 :: v_dual_mov_b32 v48, s43
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s42 :: v_dual_mov_b32 v37, s41
-; GFX11-FAKE16-NEXT: .LBB49_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s41 :: v_dual_mov_b32 v37, s40
+; GFX11-FAKE16-NEXT: .LBB49_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v33, 0xffff, v33
; GFX11-FAKE16-NEXT: v_and_b32_e32 v31, 0xffff, v31
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -39465,10 +40161,8 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_writelane_b32 v30, s87, 29
; SI-NEXT: v_writelane_b32 v30, s96, 30
; SI-NEXT: v_writelane_b32 v30, s97, 31
-; SI-NEXT: v_writelane_b32 v30, s98, 32
-; SI-NEXT: v_writelane_b32 v30, s99, 33
-; SI-NEXT: v_writelane_b32 v30, s30, 34
-; SI-NEXT: v_writelane_b32 v30, s31, 35
+; SI-NEXT: v_writelane_b32 v30, s30, 32
+; SI-NEXT: v_writelane_b32 v30, s31, 33
; SI-NEXT: v_readfirstlane_b32 s7, v15
; SI-NEXT: v_readfirstlane_b32 s9, v14
; SI-NEXT: v_readfirstlane_b32 s11, v13
@@ -39480,25 +40174,25 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s79, v7
; SI-NEXT: v_readfirstlane_b32 s89, v6
; SI-NEXT: v_readfirstlane_b32 s93, v5
-; SI-NEXT: v_readfirstlane_b32 s30, v4
-; SI-NEXT: v_readfirstlane_b32 s35, v3
-; SI-NEXT: v_readfirstlane_b32 s70, v2
-; SI-NEXT: v_readfirstlane_b32 s81, v1
-; SI-NEXT: v_readfirstlane_b32 s84, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v4
+; SI-NEXT: v_readfirstlane_b32 s31, v3
+; SI-NEXT: v_readfirstlane_b32 s68, v2
+; SI-NEXT: v_readfirstlane_b32 s71, v1
+; SI-NEXT: v_readfirstlane_b32 s82, v0
; SI-NEXT: s_lshr_b32 s90, s29, 16
; SI-NEXT: s_lshr_b32 s92, s28, 16
; SI-NEXT: s_lshr_b32 s94, s27, 16
-; SI-NEXT: s_lshr_b32 s31, s26, 16
-; SI-NEXT: s_lshr_b32 s68, s25, 16
-; SI-NEXT: s_lshr_b32 s71, s24, 16
-; SI-NEXT: s_lshr_b32 s82, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s26, 16
+; SI-NEXT: s_lshr_b32 s34, s25, 16
+; SI-NEXT: s_lshr_b32 s69, s24, 16
+; SI-NEXT: s_lshr_b32 s80, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s6, s7, 16
; SI-NEXT: s_lshr_b32 s8, s9, 16
; SI-NEXT: s_lshr_b32 s10, s11, 16
@@ -39510,47 +40204,47 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_lshr_b32 s78, s79, 16
; SI-NEXT: s_lshr_b32 s88, s89, 16
; SI-NEXT: s_lshr_b32 s91, s93, 16
-; SI-NEXT: s_lshr_b32 s95, s30, 16
-; SI-NEXT: s_lshr_b32 s34, s35, 16
-; SI-NEXT: s_lshr_b32 s69, s70, 16
-; SI-NEXT: s_lshr_b32 s80, s81, 16
-; SI-NEXT: s_lshr_b32 s83, s84, 16
+; SI-NEXT: s_lshr_b32 s95, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s30, s31, 16
+; SI-NEXT: s_lshr_b32 s35, s68, 16
+; SI-NEXT: s_lshr_b32 s70, s71, 16
+; SI-NEXT: s_lshr_b32 s81, s82, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_4
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
; SI-NEXT: s_lshl_b32 s5, s94, 16
@@ -39561,19 +40255,19 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s52, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s53, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s54, s4, s5
; SI-NEXT: s_and_b32 s4, s93, 0xffff
@@ -39609,61 +40303,70 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s7, 0xffff
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB51_3
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s36, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s96, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s37, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s38, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s39, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s22, s22, 3
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s26, s26, 3
; SI-NEXT: s_add_i32 s45, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
@@ -39680,29 +40383,29 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s84, s84, 3
+; SI-NEXT: s_add_i32 s82, s82, 3
; SI-NEXT: s_add_i32 s49, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s84, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_and_b32 s4, s82, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s81, s81, 3
+; SI-NEXT: s_add_i32 s71, s71, 3
; SI-NEXT: s_add_i32 s50, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s81, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s71, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s70, s70, 3
+; SI-NEXT: s_add_i32 s68, s68, 3
; SI-NEXT: s_add_i32 s51, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s35, s35, 3
+; SI-NEXT: s_add_i32 s31, s31, 3
; SI-NEXT: s_add_i32 s52, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s30, s30, 3
+; SI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
; SI-NEXT: s_add_i32 s53, s4, 0x30000
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s93, s93, 3
@@ -39761,8 +40464,8 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_lshl_b32 s5, s6, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s4, 0x30000
-; SI-NEXT: .LBB51_3: ; %end
-; SI-NEXT: v_readlane_b32 s30, v30, 34
+; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: v_readlane_b32 s30, v30, 32
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -39793,9 +40496,7 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: v_mov_b32_e32 v27, s63
; SI-NEXT: v_mov_b32_e32 v28, s64
; SI-NEXT: v_mov_b32_e32 v29, s65
-; SI-NEXT: v_readlane_b32 s31, v30, 35
-; SI-NEXT: v_readlane_b32 s99, v30, 33
-; SI-NEXT: v_readlane_b32 s98, v30, 32
+; SI-NEXT: v_readlane_b32 s31, v30, 33
; SI-NEXT: v_readlane_b32 s97, v30, 31
; SI-NEXT: v_readlane_b32 s96, v30, 30
; SI-NEXT: v_readlane_b32 s87, v30, 29
@@ -39833,9 +40534,6 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB51_4:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB51_2
;
; VI-LABEL: bitcast_v60i16_to_v15f64_scalar:
; VI: ; %bb.0:
@@ -39876,312 +40574,399 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; VI-NEXT: v_writelane_b32 v30, s87, 29
; VI-NEXT: v_writelane_b32 v30, s30, 30
; VI-NEXT: v_writelane_b32 v30, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s11, v13
-; VI-NEXT: s_lshr_b32 s63, s11, 16
-; VI-NEXT: v_readfirstlane_b32 s13, v12
+; VI-NEXT: s_lshr_b32 s14, s20, 16
; VI-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 s62, s13, 16
-; VI-NEXT: v_readfirstlane_b32 s15, v11
-; VI-NEXT: v_writelane_b32 v31, s63, 0
-; VI-NEXT: s_lshr_b32 s61, s15, 16
-; VI-NEXT: v_readfirstlane_b32 s73, v10
-; VI-NEXT: v_writelane_b32 v31, s62, 1
-; VI-NEXT: v_readfirstlane_b32 s7, v15
-; VI-NEXT: v_readfirstlane_b32 s9, v14
-; VI-NEXT: s_lshr_b32 s60, s73, 16
-; VI-NEXT: v_readfirstlane_b32 s75, v9
+; VI-NEXT: s_lshr_b32 s42, s19, 16
+; VI-NEXT: v_writelane_b32 v31, s14, 0
+; VI-NEXT: s_lshr_b32 s43, s18, 16
+; VI-NEXT: v_readfirstlane_b32 s6, v15
+; VI-NEXT: v_readfirstlane_b32 s8, v14
+; VI-NEXT: v_readfirstlane_b32 s10, v13
+; VI-NEXT: v_readfirstlane_b32 s12, v12
+; VI-NEXT: v_readfirstlane_b32 s87, v11
+; VI-NEXT: v_readfirstlane_b32 s15, v10
+; VI-NEXT: v_readfirstlane_b32 s72, v9
; VI-NEXT: v_readfirstlane_b32 s77, v8
; VI-NEXT: v_readfirstlane_b32 s79, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s31, v5
-; VI-NEXT: v_readfirstlane_b32 s68, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: v_writelane_b32 v31, s61, 2
-; VI-NEXT: s_lshr_b32 s90, s29, 16
-; VI-NEXT: s_lshr_b32 s30, s28, 16
-; VI-NEXT: s_lshr_b32 s34, s27, 16
-; VI-NEXT: s_lshr_b32 s69, s26, 16
-; VI-NEXT: s_lshr_b32 s80, s25, 16
-; VI-NEXT: s_lshr_b32 s83, s24, 16
-; VI-NEXT: s_lshr_b32 s86, s23, 16
-; VI-NEXT: s_lshr_b32 s8, s22, 16
-; VI-NEXT: s_lshr_b32 s10, s21, 16
-; VI-NEXT: s_lshr_b32 s12, s20, 16
-; VI-NEXT: s_lshr_b32 s14, s19, 16
-; VI-NEXT: s_lshr_b32 s72, s18, 16
-; VI-NEXT: s_lshr_b32 s74, s17, 16
-; VI-NEXT: s_lshr_b32 s76, s16, 16
-; VI-NEXT: s_lshr_b32 s66, s7, 16
-; VI-NEXT: s_lshr_b32 s67, s9, 16
-; VI-NEXT: s_lshr_b32 s59, s75, 16
-; VI-NEXT: s_lshr_b32 s58, s77, 16
-; VI-NEXT: s_lshr_b32 s78, s79, 16
-; VI-NEXT: s_lshr_b32 s88, s89, 16
-; VI-NEXT: s_lshr_b32 s91, s31, 16
-; VI-NEXT: s_lshr_b32 s35, s68, 16
-; VI-NEXT: s_lshr_b32 s70, s71, 16
-; VI-NEXT: s_lshr_b32 s81, s82, 16
-; VI-NEXT: s_lshr_b32 s84, s85, 16
-; VI-NEXT: s_lshr_b32 s87, s6, 16
+; VI-NEXT: v_readfirstlane_b32 vcc_hi, v5
+; VI-NEXT: v_readfirstlane_b32 s34, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v31, s42, 1
+; VI-NEXT: s_lshr_b32 s35, s29, 16
+; VI-NEXT: s_lshr_b32 s68, s28, 16
+; VI-NEXT: s_lshr_b32 s70, s27, 16
+; VI-NEXT: s_lshr_b32 s81, s26, 16
+; VI-NEXT: s_lshr_b32 s84, s25, 16
+; VI-NEXT: s_lshr_b32 s7, s24, 16
+; VI-NEXT: s_lshr_b32 s11, s23, 16
+; VI-NEXT: s_lshr_b32 s73, s22, 16
+; VI-NEXT: s_lshr_b32 s75, s21, 16
+; VI-NEXT: s_lshr_b32 s41, s17, 16
+; VI-NEXT: s_lshr_b32 s5, s16, 16
+; VI-NEXT: s_lshr_b32 s66, s6, 16
+; VI-NEXT: s_lshr_b32 s74, s8, 16
+; VI-NEXT: s_lshr_b32 s76, s10, 16
+; VI-NEXT: s_lshr_b32 s78, s12, 16
+; VI-NEXT: s_lshr_b32 s88, s87, 16
+; VI-NEXT: s_lshr_b32 s90, s15, 16
+; VI-NEXT: s_lshr_b32 s91, s72, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s77, 16
+; VI-NEXT: s_lshr_b32 s30, s79, 16
+; VI-NEXT: s_lshr_b32 s31, s89, 16
+; VI-NEXT: s_lshr_b32 s67, vcc_hi, 16
+; VI-NEXT: s_lshr_b32 s71, s34, 16
+; VI-NEXT: s_lshr_b32 s82, s69, 16
+; VI-NEXT: s_lshr_b32 s85, s80, 16
+; VI-NEXT: s_lshr_b32 s9, s83, 16
+; VI-NEXT: s_lshr_b32 s13, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v31, s60, 3
+; VI-NEXT: v_writelane_b32 v31, s43, 2
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v31, s59, 4
-; VI-NEXT: v_writelane_b32 v31, s58, 5
-; VI-NEXT: s_cbranch_scc0 .LBB51_4
+; VI-NEXT: v_writelane_b32 v31, s41, 3
+; VI-NEXT: v_writelane_b32 v31, s5, 4
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s43, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s42, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s74, 16
+; VI-NEXT: s_lshl_b32 s41, s41, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s14, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s68
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s34
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s31
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_hi
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s79
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s77
-; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s58, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s75
-; VI-NEXT: s_lshl_b32 s5, s59, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s72
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s59, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s73
-; VI-NEXT: s_lshl_b32 s5, s60, 16
-; VI-NEXT: s_or_b32 s60, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s15
-; VI-NEXT: s_lshl_b32 s5, s61, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_or_b32 s60, s4, s5
+; VI-NEXT: s_and_b32 s4, 0xffff, s87
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s61, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s13
-; VI-NEXT: s_lshl_b32 s5, s62, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s12
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s62, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s11
-; VI-NEXT: s_lshl_b32 s5, s63, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s10
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s63, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s9
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s8
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s64, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
+; VI-NEXT: s_and_b32 s4, 0xffff, s6
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB51_3
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: v_writelane_b32 v31, s26, 5
+; VI-NEXT: s_mov_b32 s26, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s27
+; VI-NEXT: s_mov_b32 s27, s89
+; VI-NEXT: s_mov_b32 s89, s28
+; VI-NEXT: s_mov_b32 s28, s79
+; VI-NEXT: s_mov_b32 s79, s29
+; VI-NEXT: s_mov_b32 s29, s77
+; VI-NEXT: s_mov_b32 s77, s13
+; VI-NEXT: s_mov_b32 s14, s12
+; VI-NEXT: s_mov_b32 s13, s11
+; VI-NEXT: s_mov_b32 s12, s10
+; VI-NEXT: s_mov_b32 s11, s9
+; VI-NEXT: s_mov_b32 s10, s8
+; VI-NEXT: s_mov_b32 s9, s7
+; VI-NEXT: s_mov_b32 s8, s6
+; VI-NEXT: s_mov_b32 s7, s85
+; VI-NEXT: s_mov_b32 s85, s82
+; VI-NEXT: s_mov_b32 s82, s71
+; VI-NEXT: s_mov_b32 s71, s68
+; VI-NEXT: s_mov_b32 s68, s35
+; VI-NEXT: s_mov_b32 s35, s31
+; VI-NEXT: s_mov_b32 s31, s30
+; VI-NEXT: s_mov_b32 s30, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s91
+; VI-NEXT: s_mov_b32 s91, s90
+; VI-NEXT: s_mov_b32 s90, s88
+; VI-NEXT: s_mov_b32 s88, s78
+; VI-NEXT: s_mov_b32 s78, s76
+; VI-NEXT: s_mov_b32 s76, s74
+; VI-NEXT: s_mov_b32 s74, s66
+; VI-NEXT: s_mov_b32 s6, s75
+; VI-NEXT: s_mov_b32 s75, s72
+; VI-NEXT: s_mov_b32 s72, s73
+; VI-NEXT: s_mov_b32 s73, s15
+; VI-NEXT: s_mov_b32 s15, s87
+; VI-NEXT: s_mov_b32 s87, s84
+; VI-NEXT: s_mov_b32 s84, s81
+; VI-NEXT: s_mov_b32 s81, s70
+; VI-NEXT: s_mov_b32 s70, s67
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: s_mov_b32 s67, s70
+; VI-NEXT: s_mov_b32 s70, s81
+; VI-NEXT: s_mov_b32 s81, s84
+; VI-NEXT: s_mov_b32 s84, s87
+; VI-NEXT: s_mov_b32 s87, s15
+; VI-NEXT: s_mov_b32 s15, s73
+; VI-NEXT: s_mov_b32 s73, s72
+; VI-NEXT: s_mov_b32 s72, s75
+; VI-NEXT: s_mov_b32 s75, s6
+; VI-NEXT: s_mov_b32 s66, s74
+; VI-NEXT: s_mov_b32 s74, s76
+; VI-NEXT: s_mov_b32 s76, s78
+; VI-NEXT: s_mov_b32 s78, s88
+; VI-NEXT: s_mov_b32 s88, s90
+; VI-NEXT: s_mov_b32 s90, s91
+; VI-NEXT: s_mov_b32 s91, vcc_lo
+; VI-NEXT: s_mov_b32 vcc_lo, s30
+; VI-NEXT: s_mov_b32 s30, s31
+; VI-NEXT: s_mov_b32 s31, s35
+; VI-NEXT: s_mov_b32 s35, s68
+; VI-NEXT: s_mov_b32 s68, s71
+; VI-NEXT: s_mov_b32 s71, s82
+; VI-NEXT: s_mov_b32 s82, s85
+; VI-NEXT: s_mov_b32 s85, s7
+; VI-NEXT: s_mov_b32 s6, s8
+; VI-NEXT: s_mov_b32 s7, s9
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s10, s12
+; VI-NEXT: s_mov_b32 s11, s13
+; VI-NEXT: s_mov_b32 s12, s14
+; VI-NEXT: s_mov_b32 s13, s77
+; VI-NEXT: s_mov_b32 s77, s29
+; VI-NEXT: s_mov_b32 s29, s79
+; VI-NEXT: s_mov_b32 s79, s28
+; VI-NEXT: s_mov_b32 s28, s89
+; VI-NEXT: s_mov_b32 s89, s27
+; VI-NEXT: s_mov_b32 s27, vcc_hi
+; VI-NEXT: s_mov_b32 vcc_hi, s26
+; VI-NEXT: v_readlane_b32 s26, v31, 5
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 4
; VI-NEXT: s_and_b32 s4, s16, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s76, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_add_i32 s17, s17, 3
+; VI-NEXT: v_readlane_b32 s14, v31, 3
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_and_b32 s5, s17, 0xffff
-; VI-NEXT: s_lshl_b32 s16, s74, 16
-; VI-NEXT: s_or_b32 s5, s16, s5
+; VI-NEXT: s_lshl_b32 s14, s14, 16
+; VI-NEXT: s_or_b32 s5, s14, s5
+; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_add_i32 s18, s18, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 2
; VI-NEXT: s_add_i32 s36, s4, 0x30000
-; VI-NEXT: s_add_i32 s37, s5, 0x30000
; VI-NEXT: s_and_b32 s4, s18, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s72, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s19, s19, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 1
; VI-NEXT: s_add_i32 s38, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s19, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s14, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s20, s20, 3
+; VI-NEXT: v_readlane_b32 s5, v31, 0
; VI-NEXT: s_add_i32 s39, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s20, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s12, 16
+; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s21, s21, 3
; VI-NEXT: s_add_i32 s40, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s21, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s10, 16
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s22, s22, 3
; VI-NEXT: s_add_i32 s41, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s22, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s8, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s23, s23, 3
; VI-NEXT: s_add_i32 s42, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s23, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s24, s24, 3
; VI-NEXT: s_add_i32 s43, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s24, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s25, s25, 3
; VI-NEXT: s_add_i32 s44, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s25, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s26, s26, 3
; VI-NEXT: s_add_i32 s45, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s26, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s27, s27, 3
; VI-NEXT: s_add_i32 s46, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s27, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s28, s28, 3
; VI-NEXT: s_add_i32 s47, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s28, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s30, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s48, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s29, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s90, 16
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s6, s6, 3
+; VI-NEXT: s_add_i32 s86, s86, 3
; VI-NEXT: s_add_i32 s49, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, s86, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s85, s85, 3
+; VI-NEXT: s_add_i32 s83, s83, 3
; VI-NEXT: s_add_i32 s50, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s85, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_and_b32 s4, s83, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s82, s82, 3
+; VI-NEXT: s_add_i32 s80, s80, 3
; VI-NEXT: s_add_i32 s51, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s82, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_and_b32 s4, s80, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s71, s71, 3
+; VI-NEXT: s_add_i32 s69, s69, 3
; VI-NEXT: s_add_i32 s52, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s71, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_and_b32 s4, s69, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s68, s68, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
; VI-NEXT: s_add_i32 s53, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s68, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s35, 16
+; VI-NEXT: s_and_b32 s4, s34, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s54, s4, 0x30000
-; VI-NEXT: s_and_b32 s4, s31, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s91, 16
+; VI-NEXT: s_and_b32 s4, vcc_hi, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s67, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s55, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s89, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s31, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s79, s79, 3
; VI-NEXT: s_add_i32 s56, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s79, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s78, 16
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s77, s77, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 5
; VI-NEXT: s_add_i32 s57, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s77, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s75, s75, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 4
+; VI-NEXT: s_add_i32 s75, s72, 3
; VI-NEXT: s_add_i32 s58, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s75, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s91, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s73, s73, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 3
+; VI-NEXT: s_add_i32 s73, s15, 3
; VI-NEXT: s_add_i32 s59, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s73, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s90, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s15, s15, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 2
+; VI-NEXT: s_add_i32 s15, s87, 3
; VI-NEXT: s_add_i32 s60, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s15, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s88, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s13, s13, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 1
+; VI-NEXT: s_add_i32 s13, s12, 3
; VI-NEXT: s_add_i32 s61, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s13, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s78, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s11, s11, 3
-; VI-NEXT: v_readlane_b32 s5, v31, 0
+; VI-NEXT: s_add_i32 s11, s10, 3
; VI-NEXT: s_add_i32 s62, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s11, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s5, s76, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s9, s9, 3
+; VI-NEXT: s_add_i32 s9, s8, 3
; VI-NEXT: s_add_i32 s63, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s9, 0xffff
-; VI-NEXT: s_lshl_b32 s5, s67, 16
+; VI-NEXT: s_lshl_b32 s5, s74, 16
; VI-NEXT: s_or_b32 s4, s5, s4
-; VI-NEXT: s_add_i32 s7, s7, 3
+; VI-NEXT: s_add_i32 s7, s6, 3
; VI-NEXT: s_add_i32 s64, s4, 0x30000
; VI-NEXT: s_and_b32 s4, s7, 0xffff
; VI-NEXT: s_lshl_b32 s5, s66, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_add_i32 s65, s4, 0x30000
-; VI-NEXT: .LBB51_3: ; %end
+; VI-NEXT: .LBB51_5: ; %end
; VI-NEXT: v_readlane_b32 s30, v30, 30
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
@@ -40250,13 +41035,6 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_4:
-; VI-NEXT: s_mov_b32 vcc_lo, s66
-; VI-NEXT: s_mov_b32 vcc_hi, s67
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_mov_b32 s67, vcc_hi
-; VI-NEXT: s_mov_b32 s66, vcc_lo
-; VI-NEXT: s_branch .LBB51_2
;
; GFX9-LABEL: bitcast_v60i16_to_v15f64_scalar:
; GFX9: ; %bb.0:
@@ -40356,10 +41134,18 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v0, s36, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s37, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v2, s38, 3 op_sel_hi:[1,0]
@@ -40390,10 +41176,8 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: v_pk_add_u16 v27, s63, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v28, s64, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v29, s65, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB51_5
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: s_branch .LBB51_6
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -40426,7 +41210,7 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB51_5: ; %end
+; GFX9-NEXT: .LBB51_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -40525,11 +41309,16 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
@@ -40561,8 +41350,6 @@ define inreg <15 x double> @bitcast_v60i16_to_v15f64_scalar(<60 x i16> inreg %a,
; GFX11-NEXT: v_pk_add_u16 v28, s28, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v29, s29, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -41487,7 +42274,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; SI-NEXT: v_readfirstlane_b32 s43, v1
; SI-NEXT: s_cmp_lg_u32 s42, 0
; SI-NEXT: v_readfirstlane_b32 s42, v0
-; SI-NEXT: s_cbranch_scc0 .LBB53_3
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s64, s15, 16
; SI-NEXT: s_lshr_b32 s55, s41, 16
@@ -41518,9 +42305,47 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; SI-NEXT: s_lshr_b64 s[90:91], s[22:23], 16
; SI-NEXT: s_lshr_b64 s[92:93], s[20:21], 16
; SI-NEXT: s_lshr_b64 s[94:95], s[18:19], 16
-; SI-NEXT: s_lshr_b64 s[30:31], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB53_4
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_lshr_b64 vcc, s[16:17], 16
+; SI-NEXT: s_mov_b64 s[30:31], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[30:31], -1
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr35
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr90
+; SI-NEXT: ; implicit-def: $sgpr37
+; SI-NEXT: ; implicit-def: $sgpr88
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr78
+; SI-NEXT: ; implicit-def: $sgpr39
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr74
+; SI-NEXT: ; implicit-def: $sgpr49
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr51
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr53
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr54
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $sgpr55
+; SI-NEXT: ; implicit-def: $sgpr64
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[30:31], s[30:31], exec
+; SI-NEXT: s_cselect_b32 s45, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s45, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f64 v[28:29], s[14:15], 1.0
; SI-NEXT: v_add_f64 v[26:27], s[40:41], 1.0
; SI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
@@ -41573,40 +42398,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: s_branch .LBB53_5
-; SI-NEXT: .LBB53_3:
-; SI-NEXT: ; implicit-def: $sgpr30
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr94
-; SI-NEXT: ; implicit-def: $sgpr35
-; SI-NEXT: ; implicit-def: $sgpr92
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr90
-; SI-NEXT: ; implicit-def: $sgpr37
-; SI-NEXT: ; implicit-def: $sgpr88
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: ; implicit-def: $sgpr78
-; SI-NEXT: ; implicit-def: $sgpr39
-; SI-NEXT: ; implicit-def: $sgpr76
-; SI-NEXT: ; implicit-def: $sgpr48
-; SI-NEXT: ; implicit-def: $sgpr74
-; SI-NEXT: ; implicit-def: $sgpr49
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr50
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr51
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr52
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr53
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr54
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr55
-; SI-NEXT: ; implicit-def: $sgpr64
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: s_branch .LBB53_2
-; SI-NEXT: .LBB53_4:
+; SI-NEXT: s_branch .LBB53_6
+; SI-NEXT: .LBB53_5:
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v3, s19
@@ -41659,7 +42452,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v60, s35
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v61, s34
-; SI-NEXT: v_mov_b32_e32 v53, s30
+; SI-NEXT: v_mov_b32_e32 v53, vcc_lo
; SI-NEXT: v_mov_b32_e32 v52, s94
; SI-NEXT: v_mov_b32_e32 v51, s92
; SI-NEXT: v_mov_b32_e32 v50, s90
@@ -41674,7 +42467,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; SI-NEXT: v_mov_b32_e32 v32, s56
; SI-NEXT: v_mov_b32_e32 v31, s44
; SI-NEXT: v_mov_b32_e32 v30, s46
-; SI-NEXT: .LBB53_5: ; %end
+; SI-NEXT: .LBB53_6: ; %end
; SI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v39
@@ -41824,10 +42617,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_writelane_b32 v60, s35, 1
; VI-NEXT: v_writelane_b32 v60, s36, 2
; VI-NEXT: v_writelane_b32 v60, s37, 3
-; VI-NEXT: v_writelane_b32 v60, s38, 4
-; VI-NEXT: v_writelane_b32 v60, s39, 5
-; VI-NEXT: v_writelane_b32 v60, s30, 6
-; VI-NEXT: v_writelane_b32 v60, s31, 7
+; VI-NEXT: v_writelane_b32 v60, s30, 4
+; VI-NEXT: v_writelane_b32 v60, s31, 5
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: v_readfirstlane_b32 s9, v15
; VI-NEXT: v_readfirstlane_b32 s8, v14
@@ -41846,7 +42637,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_readfirstlane_b32 s5, v1
; VI-NEXT: s_cmp_lg_u32 s4, 0
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s46, s9, 16
; VI-NEXT: s_lshr_b32 s77, s8, 16
@@ -41863,23 +42654,61 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: s_lshr_b32 s60, s7, 16
; VI-NEXT: s_lshr_b32 s91, s6, 16
; VI-NEXT: s_lshr_b32 s61, s5, 16
-; VI-NEXT: s_lshr_b32 s30, s4, 16
+; VI-NEXT: s_lshr_b32 vcc_lo, s4, 16
; VI-NEXT: s_lshr_b32 s62, s29, 16
-; VI-NEXT: s_lshr_b32 s31, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s28, 16
; VI-NEXT: s_lshr_b32 s63, s27, 16
-; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s30, s26, 16
; VI-NEXT: s_lshr_b32 s72, s25, 16
-; VI-NEXT: s_lshr_b32 s35, s24, 16
+; VI-NEXT: s_lshr_b32 s31, s24, 16
; VI-NEXT: s_lshr_b32 s73, s23, 16
-; VI-NEXT: s_lshr_b32 s36, s22, 16
+; VI-NEXT: s_lshr_b32 s34, s22, 16
; VI-NEXT: s_lshr_b32 s74, s21, 16
-; VI-NEXT: s_lshr_b32 s37, s20, 16
+; VI-NEXT: s_lshr_b32 s35, s20, 16
; VI-NEXT: s_lshr_b32 s75, s19, 16
-; VI-NEXT: s_lshr_b32 s38, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s18, 16
; VI-NEXT: s_lshr_b32 s76, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_lshr_b32 s37, s16, 16
+; VI-NEXT: s_mov_b64 s[44:45], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[44:45], -1
+; VI-NEXT: ; implicit-def: $sgpr37
+; VI-NEXT: ; implicit-def: $sgpr76
+; VI-NEXT: ; implicit-def: $sgpr36
+; VI-NEXT: ; implicit-def: $sgpr75
+; VI-NEXT: ; implicit-def: $sgpr35
+; VI-NEXT: ; implicit-def: $sgpr74
+; VI-NEXT: ; implicit-def: $sgpr34
+; VI-NEXT: ; implicit-def: $sgpr73
+; VI-NEXT: ; implicit-def: $sgpr31
+; VI-NEXT: ; implicit-def: $sgpr72
+; VI-NEXT: ; implicit-def: $sgpr30
+; VI-NEXT: ; implicit-def: $sgpr63
+; VI-NEXT: ; implicit-def: $vcc_hi
+; VI-NEXT: ; implicit-def: $sgpr62
+; VI-NEXT: ; implicit-def: $vcc_lo
+; VI-NEXT: ; implicit-def: $sgpr61
+; VI-NEXT: ; implicit-def: $sgpr91
+; VI-NEXT: ; implicit-def: $sgpr60
+; VI-NEXT: ; implicit-def: $sgpr90
+; VI-NEXT: ; implicit-def: $sgpr59
+; VI-NEXT: ; implicit-def: $sgpr89
+; VI-NEXT: ; implicit-def: $sgpr58
+; VI-NEXT: ; implicit-def: $sgpr88
+; VI-NEXT: ; implicit-def: $sgpr57
+; VI-NEXT: ; implicit-def: $sgpr79
+; VI-NEXT: ; implicit-def: $sgpr56
+; VI-NEXT: ; implicit-def: $sgpr78
+; VI-NEXT: ; implicit-def: $sgpr47
+; VI-NEXT: ; implicit-def: $sgpr77
+; VI-NEXT: ; implicit-def: $sgpr46
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; VI-NEXT: s_cselect_b32 s44, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s44, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f64 v[28:29], s[8:9], 1.0
; VI-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; VI-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
@@ -41925,40 +42754,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_lshrrev_b32_e32 v46, 16, v2
; VI-NEXT: v_lshrrev_b32_e32 v59, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v56, 16, v0
-; VI-NEXT: s_branch .LBB53_5
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: ; implicit-def: $sgpr39
-; VI-NEXT: ; implicit-def: $sgpr76
-; VI-NEXT: ; implicit-def: $sgpr38
-; VI-NEXT: ; implicit-def: $sgpr75
-; VI-NEXT: ; implicit-def: $sgpr37
-; VI-NEXT: ; implicit-def: $sgpr74
-; VI-NEXT: ; implicit-def: $sgpr36
-; VI-NEXT: ; implicit-def: $sgpr73
-; VI-NEXT: ; implicit-def: $sgpr35
-; VI-NEXT: ; implicit-def: $sgpr72
-; VI-NEXT: ; implicit-def: $sgpr34
-; VI-NEXT: ; implicit-def: $sgpr63
-; VI-NEXT: ; implicit-def: $sgpr31
-; VI-NEXT: ; implicit-def: $sgpr62
-; VI-NEXT: ; implicit-def: $sgpr30
-; VI-NEXT: ; implicit-def: $sgpr61
-; VI-NEXT: ; implicit-def: $sgpr91
-; VI-NEXT: ; implicit-def: $sgpr60
-; VI-NEXT: ; implicit-def: $sgpr90
-; VI-NEXT: ; implicit-def: $sgpr59
-; VI-NEXT: ; implicit-def: $sgpr89
-; VI-NEXT: ; implicit-def: $sgpr58
-; VI-NEXT: ; implicit-def: $sgpr88
-; VI-NEXT: ; implicit-def: $sgpr57
-; VI-NEXT: ; implicit-def: $sgpr79
-; VI-NEXT: ; implicit-def: $sgpr56
-; VI-NEXT: ; implicit-def: $sgpr78
-; VI-NEXT: ; implicit-def: $sgpr47
-; VI-NEXT: ; implicit-def: $sgpr77
-; VI-NEXT: ; implicit-def: $sgpr46
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: s_branch .LBB53_6
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: v_mov_b32_e32 v4, s20
@@ -41989,14 +42786,14 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v27, s11
; VI-NEXT: v_mov_b32_e32 v28, s8
; VI-NEXT: v_mov_b32_e32 v29, s9
-; VI-NEXT: v_mov_b32_e32 v56, s39
-; VI-NEXT: v_mov_b32_e32 v46, s38
-; VI-NEXT: v_mov_b32_e32 v44, s37
-; VI-NEXT: v_mov_b32_e32 v42, s36
-; VI-NEXT: v_mov_b32_e32 v40, s35
-; VI-NEXT: v_mov_b32_e32 v54, s34
-; VI-NEXT: v_mov_b32_e32 v52, s31
-; VI-NEXT: v_mov_b32_e32 v50, s30
+; VI-NEXT: v_mov_b32_e32 v56, s37
+; VI-NEXT: v_mov_b32_e32 v46, s36
+; VI-NEXT: v_mov_b32_e32 v44, s35
+; VI-NEXT: v_mov_b32_e32 v42, s34
+; VI-NEXT: v_mov_b32_e32 v40, s31
+; VI-NEXT: v_mov_b32_e32 v54, s30
+; VI-NEXT: v_mov_b32_e32 v52, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v50, vcc_lo
; VI-NEXT: v_mov_b32_e32 v48, s91
; VI-NEXT: v_mov_b32_e32 v38, s90
; VI-NEXT: v_mov_b32_e32 v36, s89
@@ -42019,7 +42816,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_mov_b32_e32 v37, s56
; VI-NEXT: v_mov_b32_e32 v35, s47
; VI-NEXT: v_mov_b32_e32 v33, s46
-; VI-NEXT: .LBB53_5: ; %end
+; VI-NEXT: .LBB53_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v56, 16, v56
; VI-NEXT: v_lshlrev_b32_e32 v46, 16, v46
; VI-NEXT: v_lshlrev_b32_e32 v44, 16, v44
@@ -42082,7 +42879,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v35
; VI-NEXT: v_or_b32_sdwa v28, v28, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v33
-; VI-NEXT: v_readlane_b32 s30, v60, 6
+; VI-NEXT: v_readlane_b32 s30, v60, 4
; VI-NEXT: v_or_b32_sdwa v11, v11, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v13, v13, v52 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v15, v15, v50 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -42093,9 +42890,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; VI-NEXT: v_or_b32_sdwa v25, v25, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v27, v27, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v60, 7
-; VI-NEXT: v_readlane_b32 s39, v60, 5
-; VI-NEXT: v_readlane_b32 s38, v60, 4
+; VI-NEXT: v_readlane_b32 s31, v60, 5
; VI-NEXT: v_readlane_b32 s37, v60, 3
; VI-NEXT: v_readlane_b32 s36, v60, 2
; VI-NEXT: v_readlane_b32 s35, v60, 1
@@ -42124,10 +42919,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: v_readfirstlane_b32 s9, v15
; GFX9-NEXT: v_readfirstlane_b32 s8, v14
@@ -42146,7 +42939,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s46, s9, 16
; GFX9-NEXT: s_lshr_b32 s77, s8, 16
@@ -42171,15 +42964,53 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: s_lshr_b32 s72, s25, 16
; GFX9-NEXT: s_lshr_b32 s95, s24, 16
; GFX9-NEXT: s_lshr_b32 s73, s23, 16
-; GFX9-NEXT: s_lshr_b32 s30, s22, 16
+; GFX9-NEXT: s_lshr_b32 vcc_lo, s22, 16
; GFX9-NEXT: s_lshr_b32 s74, s21, 16
-; GFX9-NEXT: s_lshr_b32 s31, s20, 16
+; GFX9-NEXT: s_lshr_b32 vcc_hi, s20, 16
; GFX9-NEXT: s_lshr_b32 s75, s19, 16
-; GFX9-NEXT: s_lshr_b32 s34, s18, 16
+; GFX9-NEXT: s_lshr_b32 s30, s18, 16
; GFX9-NEXT: s_lshr_b32 s76, s17, 16
-; GFX9-NEXT: s_lshr_b32 s35, s16, 16
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_lshr_b32 s31, s16, 16
+; GFX9-NEXT: s_mov_b64 s[44:45], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[44:45], -1
+; GFX9-NEXT: ; implicit-def: $sgpr31
+; GFX9-NEXT: ; implicit-def: $sgpr76
+; GFX9-NEXT: ; implicit-def: $sgpr30
+; GFX9-NEXT: ; implicit-def: $sgpr75
+; GFX9-NEXT: ; implicit-def: $vcc_hi
+; GFX9-NEXT: ; implicit-def: $sgpr74
+; GFX9-NEXT: ; implicit-def: $vcc_lo
+; GFX9-NEXT: ; implicit-def: $sgpr73
+; GFX9-NEXT: ; implicit-def: $sgpr95
+; GFX9-NEXT: ; implicit-def: $sgpr72
+; GFX9-NEXT: ; implicit-def: $sgpr94
+; GFX9-NEXT: ; implicit-def: $sgpr63
+; GFX9-NEXT: ; implicit-def: $sgpr93
+; GFX9-NEXT: ; implicit-def: $sgpr62
+; GFX9-NEXT: ; implicit-def: $sgpr92
+; GFX9-NEXT: ; implicit-def: $sgpr61
+; GFX9-NEXT: ; implicit-def: $sgpr91
+; GFX9-NEXT: ; implicit-def: $sgpr60
+; GFX9-NEXT: ; implicit-def: $sgpr90
+; GFX9-NEXT: ; implicit-def: $sgpr59
+; GFX9-NEXT: ; implicit-def: $sgpr89
+; GFX9-NEXT: ; implicit-def: $sgpr58
+; GFX9-NEXT: ; implicit-def: $sgpr88
+; GFX9-NEXT: ; implicit-def: $sgpr57
+; GFX9-NEXT: ; implicit-def: $sgpr79
+; GFX9-NEXT: ; implicit-def: $sgpr56
+; GFX9-NEXT: ; implicit-def: $sgpr78
+; GFX9-NEXT: ; implicit-def: $sgpr47
+; GFX9-NEXT: ; implicit-def: $sgpr77
+; GFX9-NEXT: ; implicit-def: $sgpr46
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[44:45], s[44:45], exec
+; GFX9-NEXT: s_cselect_b32 s44, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s44, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f64 v[28:29], s[8:9], 1.0
; GFX9-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX9-NEXT: v_add_f64 v[24:25], s[12:13], 1.0
@@ -42225,40 +43056,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 16, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v56, 16, v0
-; GFX9-NEXT: s_branch .LBB53_5
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: ; implicit-def: $sgpr35
-; GFX9-NEXT: ; implicit-def: $sgpr76
-; GFX9-NEXT: ; implicit-def: $sgpr34
-; GFX9-NEXT: ; implicit-def: $sgpr75
-; GFX9-NEXT: ; implicit-def: $sgpr31
-; GFX9-NEXT: ; implicit-def: $sgpr74
-; GFX9-NEXT: ; implicit-def: $sgpr30
-; GFX9-NEXT: ; implicit-def: $sgpr73
-; GFX9-NEXT: ; implicit-def: $sgpr95
-; GFX9-NEXT: ; implicit-def: $sgpr72
-; GFX9-NEXT: ; implicit-def: $sgpr94
-; GFX9-NEXT: ; implicit-def: $sgpr63
-; GFX9-NEXT: ; implicit-def: $sgpr93
-; GFX9-NEXT: ; implicit-def: $sgpr62
-; GFX9-NEXT: ; implicit-def: $sgpr92
-; GFX9-NEXT: ; implicit-def: $sgpr61
-; GFX9-NEXT: ; implicit-def: $sgpr91
-; GFX9-NEXT: ; implicit-def: $sgpr60
-; GFX9-NEXT: ; implicit-def: $sgpr90
-; GFX9-NEXT: ; implicit-def: $sgpr59
-; GFX9-NEXT: ; implicit-def: $sgpr89
-; GFX9-NEXT: ; implicit-def: $sgpr58
-; GFX9-NEXT: ; implicit-def: $sgpr88
-; GFX9-NEXT: ; implicit-def: $sgpr57
-; GFX9-NEXT: ; implicit-def: $sgpr79
-; GFX9-NEXT: ; implicit-def: $sgpr56
-; GFX9-NEXT: ; implicit-def: $sgpr78
-; GFX9-NEXT: ; implicit-def: $sgpr47
-; GFX9-NEXT: ; implicit-def: $sgpr77
-; GFX9-NEXT: ; implicit-def: $sgpr46
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: s_branch .LBB53_6
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: v_mov_b32_e32 v4, s20
@@ -42289,10 +43088,10 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v27, s11
; GFX9-NEXT: v_mov_b32_e32 v28, s8
; GFX9-NEXT: v_mov_b32_e32 v29, s9
-; GFX9-NEXT: v_mov_b32_e32 v56, s35
-; GFX9-NEXT: v_mov_b32_e32 v46, s34
-; GFX9-NEXT: v_mov_b32_e32 v44, s31
-; GFX9-NEXT: v_mov_b32_e32 v42, s30
+; GFX9-NEXT: v_mov_b32_e32 v56, s31
+; GFX9-NEXT: v_mov_b32_e32 v46, s30
+; GFX9-NEXT: v_mov_b32_e32 v44, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v42, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v40, s95
; GFX9-NEXT: v_mov_b32_e32 v54, s94
; GFX9-NEXT: v_mov_b32_e32 v52, s93
@@ -42319,7 +43118,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v37, s56
; GFX9-NEXT: v_mov_b32_e32 v35, s47
; GFX9-NEXT: v_mov_b32_e32 v33, s46
-; GFX9-NEXT: .LBB53_5: ; %end
+; GFX9-NEXT: .LBB53_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -42374,7 +43173,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v10, v54, 16, v10
; GFX9-NEXT: v_lshl_or_b32 v12, v52, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v14, v50, 16, v14
@@ -42393,9 +43192,7 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX9-NEXT: v_lshl_or_b32 v27, v35, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v30, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v33, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -42419,12 +43216,12 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s40, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s15, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s40, s15, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s72, s14, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s42, s11, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s41, s11, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s73, s10, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s43, s9, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s74, s8, 16
@@ -42452,9 +43249,46 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-TRUE16-NEXT: s_lshr_b32 s93, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s63, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB53_3
+; GFX11-TRUE16-NEXT: .LBB53_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s42, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-TRUE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[28:29], s[14:15], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[26:27], s[10:11], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[24:25], s[8:9], 1.0
@@ -42500,40 +43334,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 16, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB53_5
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
-; GFX11-TRUE16-NEXT: .LBB53_4:
+; GFX11-TRUE16-NEXT: s_branch .LBB53_6
+; GFX11-TRUE16-NEXT: .LBB53_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v5, s17
@@ -42562,9 +43364,9 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s76 :: v_dual_mov_b32 v49, s45
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s75 :: v_dual_mov_b32 v39, s44
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s74 :: v_dual_mov_b32 v37, s43
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s72 :: v_dual_mov_b32 v35, s42
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v82, s62 :: v_dual_mov_b32 v33, s41
-; GFX11-TRUE16-NEXT: .LBB53_5: ; %end
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s72 :: v_dual_mov_b32 v35, s41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v82, s62 :: v_dual_mov_b32 v33, s40
+; GFX11-TRUE16-NEXT: .LBB53_6: ; %end
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v80.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v83.l
@@ -42616,12 +43418,12 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v0
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s40, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s40, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s15, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s40, s15, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s72, s14, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s42, s11, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s41, s11, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s73, s10, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s43, s9, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s74, s8, 16
@@ -42649,9 +43451,46 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-FAKE16-NEXT: s_lshr_b32 s93, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s63, s1, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s94, s0, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB53_3
+; GFX11-FAKE16-NEXT: .LBB53_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s42, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr40
+; GFX11-FAKE16-NEXT: .LBB53_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s42, s42, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s42, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s42, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: v_add_f64 v[25:26], s[14:15], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[27:28], s[10:11], 1.0
; GFX11-FAKE16-NEXT: v_add_f64 v[29:30], s[8:9], 1.0
@@ -42697,40 +43536,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v71, 16, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v83, 16, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v81, 16, v2
-; GFX11-FAKE16-NEXT: s_branch .LBB53_5
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr94
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr63
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr93
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr62
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr92
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr61
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr91
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr60
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr90
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr59
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr89
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr58
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr88
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr57
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr79
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr56
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr78
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr47
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr77
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr46
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr76
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr45
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr75
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr44
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr74
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr43
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr73
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr42
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr72
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr41
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
-; GFX11-FAKE16-NEXT: .LBB53_4:
+; GFX11-FAKE16-NEXT: s_branch .LBB53_6
+; GFX11-FAKE16-NEXT: .LBB53_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v33, s16
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s18
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s20 :: v_dual_mov_b32 v12, s22
@@ -42760,8 +43567,8 @@ define inreg <60 x half> @bitcast_v15f64_to_v60f16_scalar(<15 x double> inreg %a
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v67, s58 :: v_dual_mov_b32 v54, s47
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, s46 :: v_dual_mov_b32 v50, s44
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, s45 :: v_dual_mov_b32 v48, s43
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s42 :: v_dual_mov_b32 v37, s41
-; GFX11-FAKE16-NEXT: .LBB53_5: ; %end
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, s41 :: v_dual_mov_b32 v37, s40
+; GFX11-FAKE16-NEXT: .LBB53_6: ; %end
; GFX11-FAKE16-NEXT: v_and_b32_e32 v33, 0xffff, v33
; GFX11-FAKE16-NEXT: v_and_b32_e32 v31, 0xffff, v31
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -44369,10 +45176,8 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_writelane_b32 v32, s87, 29
; SI-NEXT: v_writelane_b32 v32, s96, 30
; SI-NEXT: v_writelane_b32 v32, s97, 31
-; SI-NEXT: v_writelane_b32 v32, s98, 32
-; SI-NEXT: v_writelane_b32 v32, s99, 33
-; SI-NEXT: v_writelane_b32 v32, s30, 34
-; SI-NEXT: v_writelane_b32 v32, s31, 35
+; SI-NEXT: v_writelane_b32 v32, s30, 32
+; SI-NEXT: v_writelane_b32 v32, s31, 33
; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: v_readfirstlane_b32 s8, v14
; SI-NEXT: v_readfirstlane_b32 s10, v13
@@ -44386,23 +45191,23 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_readfirstlane_b32 s90, v5
; SI-NEXT: v_readfirstlane_b32 s92, v4
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: v_readfirstlane_b32 s30, v2
-; SI-NEXT: v_readfirstlane_b32 s35, v1
-; SI-NEXT: v_readfirstlane_b32 s70, v0
-; SI-NEXT: s_lshr_b32 s31, s29, 16
-; SI-NEXT: s_lshr_b32 s68, s28, 16
-; SI-NEXT: s_lshr_b32 s71, s27, 16
-; SI-NEXT: s_lshr_b32 s81, s26, 16
-; SI-NEXT: s_lshr_b32 s82, s25, 16
-; SI-NEXT: s_lshr_b32 s83, s24, 16
-; SI-NEXT: s_lshr_b32 s84, s23, 16
-; SI-NEXT: s_lshr_b32 s85, s22, 16
-; SI-NEXT: s_lshr_b32 s86, s21, 16
-; SI-NEXT: s_lshr_b32 s87, s20, 16
-; SI-NEXT: s_lshr_b32 s96, s19, 16
-; SI-NEXT: s_lshr_b32 s97, s18, 16
-; SI-NEXT: s_lshr_b32 s98, s17, 16
-; SI-NEXT: s_lshr_b32 s99, s16, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
+; SI-NEXT: v_readfirstlane_b32 s31, v1
+; SI-NEXT: v_readfirstlane_b32 s68, v0
+; SI-NEXT: s_lshr_b32 vcc_hi, s29, 16
+; SI-NEXT: s_lshr_b32 s34, s28, 16
+; SI-NEXT: s_lshr_b32 s69, s27, 16
+; SI-NEXT: s_lshr_b32 s71, s26, 16
+; SI-NEXT: s_lshr_b32 s80, s25, 16
+; SI-NEXT: s_lshr_b32 s81, s24, 16
+; SI-NEXT: s_lshr_b32 s82, s23, 16
+; SI-NEXT: s_lshr_b32 s83, s22, 16
+; SI-NEXT: s_lshr_b32 s84, s21, 16
+; SI-NEXT: s_lshr_b32 s85, s20, 16
+; SI-NEXT: s_lshr_b32 s86, s19, 16
+; SI-NEXT: s_lshr_b32 s87, s18, 16
+; SI-NEXT: s_lshr_b32 s96, s17, 16
+; SI-NEXT: s_lshr_b32 s97, s16, 16
; SI-NEXT: s_lshr_b32 s7, s6, 16
; SI-NEXT: s_lshr_b32 s9, s8, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
@@ -44416,63 +45221,63 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: s_lshr_b32 s91, s90, 16
; SI-NEXT: s_lshr_b32 s93, s92, 16
; SI-NEXT: s_lshr_b32 s95, s94, 16
-; SI-NEXT: s_lshr_b32 s34, s30, 16
-; SI-NEXT: s_lshr_b32 s69, s35, 16
-; SI-NEXT: s_lshr_b32 s80, s70, 16
+; SI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; SI-NEXT: s_lshr_b32 s35, s31, 16
+; SI-NEXT: s_lshr_b32 s70, s68, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_3
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s99, 16
+; SI-NEXT: s_lshl_b32 s5, s97, 16
; SI-NEXT: s_or_b32 s36, s4, s5
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s97, 16
+; SI-NEXT: s_lshl_b32 s5, s87, 16
; SI-NEXT: s_or_b32 s38, s4, s5
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s96, 16
+; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s41, s98, 16
+; SI-NEXT: s_lshl_b32 s41, s96, 16
; SI-NEXT: s_or_b32 s39, s4, s5
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s87, 16
+; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s37, s40, s41
; SI-NEXT: s_or_b32 s40, s4, s5
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s86, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s41, s4, s5
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s85, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s42, s4, s5
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 16
; SI-NEXT: s_or_b32 s43, s4, s5
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s81, 16
; SI-NEXT: s_or_b32 s44, s4, s5
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s80, 16
; SI-NEXT: s_or_b32 s45, s4, s5
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s46, s4, s5
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s69, 16
; SI-NEXT: s_or_b32 s47, s4, s5
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s68, 16
+; SI-NEXT: s_lshl_b32 s5, s34, 16
; SI-NEXT: s_or_b32 s48, s4, s5
; SI-NEXT: s_and_b32 s4, s29, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s31, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s49, s4, s5
-; SI-NEXT: s_and_b32 s4, s70, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_and_b32 s4, s68, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s70, 16
; SI-NEXT: s_or_b32 s50, s4, s5
-; SI-NEXT: s_and_b32 s4, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s69, 16
+; SI-NEXT: s_and_b32 s4, s31, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s35, 16
; SI-NEXT: s_or_b32 s51, s4, s5
-; SI-NEXT: s_and_b32 s4, s30, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s34, 16
+; SI-NEXT: s_and_b32 s4, vcc_lo, 0xffff
+; SI-NEXT: s_lshl_b32 s5, s30, 16
; SI-NEXT: s_or_b32 s52, s4, s5
; SI-NEXT: s_and_b32 s4, s94, 0xffff
; SI-NEXT: s_lshl_b32 s5, s95, 16
@@ -44513,11 +45318,20 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: s_and_b32 s4, s6, 0xffff
; SI-NEXT: s_lshl_b32 s5, s7, 16
; SI-NEXT: s_or_b32 s65, s4, s5
-; SI-NEXT: s_cbranch_execnz .LBB55_4
-; SI-NEXT: .LBB55_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s99
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s97
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s98
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s96
; SI-NEXT: v_cvt_f32_f16_e32 v3, s17
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -44527,7 +45341,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s97
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s87
; SI-NEXT: v_add_f32_e32 v3, 0x38000000, v3
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f32_f16_e32 v5, s19
@@ -44537,20 +45351,20 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v2, s18
; SI-NEXT: v_or_b32_e32 v1, v3, v1
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v4, s96
+; SI-NEXT: v_cvt_f32_f16_e32 v4, s86
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s87
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v5, v3
; SI-NEXT: v_cvt_f32_f16_e32 v4, s20
; SI-NEXT: v_add_f32_e32 v5, 0x38000000, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s86
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s84
; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
; SI-NEXT: v_add_f32_e32 v4, 0x38000000, v4
; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
@@ -44560,7 +45374,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v7, s21
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; SI-NEXT: v_cvt_f32_f16_e32 v6, s85
+; SI-NEXT: v_cvt_f32_f16_e32 v6, s83
; SI-NEXT: v_cvt_f32_f16_e32 v8, s22
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
@@ -44569,11 +45383,11 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
; SI-NEXT: v_cvt_f16_f32_e32 v8, v8
; SI-NEXT: v_or_b32_e32 v5, v7, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v7, s84
+; SI-NEXT: v_cvt_f32_f16_e32 v7, s82
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; SI-NEXT: v_or_b32_e32 v6, v8, v6
; SI-NEXT: v_cvt_f32_f16_e32 v8, s23
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s83
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s81
; SI-NEXT: v_add_f32_e32 v7, 0x38000000, v7
; SI-NEXT: v_cvt_f16_f32_e32 v7, v7
; SI-NEXT: v_add_f32_e32 v8, 0x38000000, v8
@@ -44584,7 +45398,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v10, s24
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v9, s82
+; SI-NEXT: v_cvt_f32_f16_e32 v9, s80
; SI-NEXT: v_cvt_f32_f16_e32 v11, s25
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
@@ -44593,11 +45407,11 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
; SI-NEXT: v_cvt_f16_f32_e32 v11, v11
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v10, s81
+; SI-NEXT: v_cvt_f32_f16_e32 v10, s71
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; SI-NEXT: v_or_b32_e32 v9, v11, v9
; SI-NEXT: v_cvt_f32_f16_e32 v11, s26
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s71
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s69
; SI-NEXT: v_add_f32_e32 v10, 0x38000000, v10
; SI-NEXT: v_cvt_f16_f32_e32 v10, v10
; SI-NEXT: v_add_f32_e32 v11, 0x38000000, v11
@@ -44608,7 +45422,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f32_f16_e32 v13, s27
; SI-NEXT: v_or_b32_e32 v10, v11, v10
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v12
-; SI-NEXT: v_cvt_f32_f16_e32 v12, s68
+; SI-NEXT: v_cvt_f32_f16_e32 v12, s34
; SI-NEXT: v_cvt_f32_f16_e32 v14, s28
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
@@ -44617,11 +45431,11 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_or_b32_e32 v11, v13, v11
-; SI-NEXT: v_cvt_f32_f16_e32 v13, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v13, vcc_hi
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; SI-NEXT: v_or_b32_e32 v12, v14, v12
; SI-NEXT: v_cvt_f32_f16_e32 v14, s29
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s80
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s70
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
@@ -44629,11 +45443,11 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_cvt_f16_f32_e32 v14, v14
; SI-NEXT: v_cvt_f16_f32_e32 v15, v15
; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s70
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s68
; SI-NEXT: v_or_b32_e32 v13, v14, v13
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v15, s69
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v15, s35
+; SI-NEXT: v_cvt_f32_f16_e32 v17, s31
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
; SI-NEXT: v_add_f32_e32 v15, 0x38000000, v15
@@ -44641,10 +45455,10 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_add_f32_e32 v17, 0x38000000, v17
; SI-NEXT: v_cvt_f16_f32_e32 v17, v17
; SI-NEXT: v_or_b32_e32 v14, v16, v14
-; SI-NEXT: v_cvt_f32_f16_e32 v16, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v16, s30
; SI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; SI-NEXT: v_or_b32_e32 v15, v17, v15
-; SI-NEXT: v_cvt_f32_f16_e32 v17, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v17, vcc_lo
; SI-NEXT: v_cvt_f32_f16_e32 v18, s95
; SI-NEXT: v_add_f32_e32 v16, 0x38000000, v16
; SI-NEXT: v_cvt_f16_f32_e32 v16, v16
@@ -44755,11 +45569,8 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_or_b32_e32 v28, v29, v28
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v30
; SI-NEXT: v_or_b32_e32 v29, v31, v29
-; SI-NEXT: s_branch .LBB55_5
-; SI-NEXT: .LBB55_3:
-; SI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; SI-NEXT: s_branch .LBB55_2
-; SI-NEXT: .LBB55_4:
+; SI-NEXT: s_branch .LBB55_6
+; SI-NEXT: .LBB55_5:
; SI-NEXT: v_mov_b32_e32 v0, s36
; SI-NEXT: v_mov_b32_e32 v1, s37
; SI-NEXT: v_mov_b32_e32 v2, s38
@@ -44792,11 +45603,9 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; SI-NEXT: v_mov_b32_e32 v29, s65
; SI-NEXT: v_mov_b32_e32 v30, s66
; SI-NEXT: v_mov_b32_e32 v31, s67
-; SI-NEXT: .LBB55_5: ; %end
-; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s31, v32, 35
-; SI-NEXT: v_readlane_b32 s99, v32, 33
-; SI-NEXT: v_readlane_b32 s98, v32, 32
+; SI-NEXT: .LBB55_6: ; %end
+; SI-NEXT: v_readlane_b32 s30, v32, 32
+; SI-NEXT: v_readlane_b32 s31, v32, 33
; SI-NEXT: v_readlane_b32 s97, v32, 31
; SI-NEXT: v_readlane_b32 s96, v32, 30
; SI-NEXT: v_readlane_b32 s87, v32, 29
@@ -44874,136 +45683,136 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; VI-NEXT: v_writelane_b32 v32, s87, 29
; VI-NEXT: v_writelane_b32 v32, s30, 30
; VI-NEXT: v_writelane_b32 v32, s31, 31
-; VI-NEXT: v_readfirstlane_b32 s6, v15
-; VI-NEXT: s_lshr_b32 vcc_lo, s6, 16
; VI-NEXT: v_readfirstlane_b32 s8, v14
-; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
-; VI-NEXT: s_lshr_b32 vcc_hi, s8, 16
+; VI-NEXT: s_lshr_b32 s75, s8, 16
; VI-NEXT: v_readfirstlane_b32 s10, v13
-; VI-NEXT: v_writelane_b32 v33, vcc_lo, 0
+; VI-NEXT: ; implicit-def: $vgpr33 : SGPR spill to VGPR lane
; VI-NEXT: s_lshr_b32 s63, s10, 16
; VI-NEXT: v_readfirstlane_b32 s12, v12
-; VI-NEXT: v_writelane_b32 v33, vcc_hi, 1
+; VI-NEXT: v_writelane_b32 v33, s75, 0
; VI-NEXT: s_lshr_b32 s62, s12, 16
; VI-NEXT: v_readfirstlane_b32 s14, v11
-; VI-NEXT: v_writelane_b32 v33, s63, 2
+; VI-NEXT: v_writelane_b32 v33, s63, 1
+; VI-NEXT: v_readfirstlane_b32 s6, v15
; VI-NEXT: s_lshr_b32 s61, s14, 16
; VI-NEXT: v_readfirstlane_b32 s72, v10
-; VI-NEXT: v_writelane_b32 v33, s62, 3
-; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: v_readfirstlane_b32 s74, v9
; VI-NEXT: v_readfirstlane_b32 s76, v8
; VI-NEXT: v_readfirstlane_b32 s78, v7
; VI-NEXT: v_readfirstlane_b32 s89, v6
-; VI-NEXT: v_readfirstlane_b32 s30, v5
-; VI-NEXT: v_readfirstlane_b32 s35, v4
-; VI-NEXT: v_readfirstlane_b32 s71, v3
-; VI-NEXT: v_readfirstlane_b32 s82, v2
-; VI-NEXT: v_readfirstlane_b32 s85, v1
-; VI-NEXT: v_readfirstlane_b32 s7, v0
-; VI-NEXT: v_writelane_b32 v33, s61, 4
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v5
+; VI-NEXT: v_readfirstlane_b32 s31, v4
+; VI-NEXT: v_readfirstlane_b32 s69, v3
+; VI-NEXT: v_readfirstlane_b32 s80, v2
+; VI-NEXT: v_readfirstlane_b32 s83, v1
+; VI-NEXT: v_readfirstlane_b32 s86, v0
+; VI-NEXT: v_writelane_b32 v33, s62, 2
; VI-NEXT: s_lshr_b32 s56, s29, 16
-; VI-NEXT: s_lshr_b32 s88, s28, 16
-; VI-NEXT: s_lshr_b32 s31, s27, 16
-; VI-NEXT: s_lshr_b32 s68, s26, 16
-; VI-NEXT: s_lshr_b32 s70, s25, 16
-; VI-NEXT: s_lshr_b32 s81, s24, 16
-; VI-NEXT: s_lshr_b32 s84, s23, 16
-; VI-NEXT: s_lshr_b32 s86, s22, 16
-; VI-NEXT: s_lshr_b32 s9, s21, 16
-; VI-NEXT: s_lshr_b32 s13, s20, 16
-; VI-NEXT: s_lshr_b32 s15, s19, 16
-; VI-NEXT: s_lshr_b32 s73, s18, 16
-; VI-NEXT: s_lshr_b32 s75, s17, 16
-; VI-NEXT: s_lshr_b32 s77, s16, 16
+; VI-NEXT: s_lshr_b32 s77, s28, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; VI-NEXT: s_lshr_b32 s34, s26, 16
+; VI-NEXT: s_lshr_b32 s68, s25, 16
+; VI-NEXT: s_lshr_b32 s71, s24, 16
+; VI-NEXT: s_lshr_b32 s82, s23, 16
+; VI-NEXT: s_lshr_b32 s84, s22, 16
+; VI-NEXT: s_lshr_b32 s87, s21, 16
+; VI-NEXT: s_lshr_b32 s9, s20, 16
+; VI-NEXT: s_lshr_b32 s11, s19, 16
+; VI-NEXT: s_lshr_b32 s13, s18, 16
+; VI-NEXT: s_lshr_b32 s15, s17, 16
+; VI-NEXT: s_lshr_b32 s73, s16, 16
+; VI-NEXT: s_lshr_b32 s65, s6, 16
+; VI-NEXT: s_lshr_b32 s60, s72, 16
; VI-NEXT: s_lshr_b32 s59, s74, 16
; VI-NEXT: s_lshr_b32 s58, s76, 16
; VI-NEXT: s_lshr_b32 s57, s78, 16
-; VI-NEXT: s_lshr_b32 s64, s89, 16
-; VI-NEXT: s_lshr_b32 s34, s30, 16
-; VI-NEXT: s_lshr_b32 s69, s35, 16
-; VI-NEXT: s_lshr_b32 s80, s71, 16
-; VI-NEXT: s_lshr_b32 s83, s82, 16
-; VI-NEXT: s_lshr_b32 s87, s85, 16
-; VI-NEXT: s_lshr_b32 s11, s7, 16
+; VI-NEXT: s_lshr_b32 s88, s89, 16
+; VI-NEXT: s_lshr_b32 s30, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s35, s31, 16
+; VI-NEXT: s_lshr_b32 s70, s69, 16
+; VI-NEXT: s_lshr_b32 s81, s80, 16
+; VI-NEXT: s_lshr_b32 s85, s83, 16
+; VI-NEXT: s_lshr_b32 s7, s86, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
-; VI-NEXT: v_writelane_b32 v33, s60, 5
+; VI-NEXT: v_writelane_b32 v33, s61, 3
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: v_writelane_b32 v33, s59, 6
-; VI-NEXT: v_writelane_b32 v33, s58, 7
-; VI-NEXT: s_cbranch_scc0 .LBB55_3
+; VI-NEXT: v_writelane_b32 v33, s60, 4
+; VI-NEXT: v_writelane_b32 v33, s59, 5
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s77, 16
+; VI-NEXT: s_lshl_b32 s5, s73, 16
; VI-NEXT: s_or_b32 s36, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s5, s73, 16
+; VI-NEXT: s_lshl_b32 s5, s13, 16
; VI-NEXT: s_or_b32 s38, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s5, s15, 16
+; VI-NEXT: s_lshl_b32 s5, s11, 16
; VI-NEXT: s_and_b32 s40, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s41, s75, 16
+; VI-NEXT: s_lshl_b32 s41, s15, 16
; VI-NEXT: s_or_b32 s39, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s20
-; VI-NEXT: s_lshl_b32 s5, s13, 16
+; VI-NEXT: s_lshl_b32 s5, s9, 16
; VI-NEXT: s_or_b32 s37, s40, s41
; VI-NEXT: s_or_b32 s40, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s21
-; VI-NEXT: s_lshl_b32 s5, s9, 16
+; VI-NEXT: s_lshl_b32 s5, s87, 16
; VI-NEXT: s_or_b32 s41, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s22
-; VI-NEXT: s_lshl_b32 s5, s86, 16
+; VI-NEXT: s_lshl_b32 s5, s84, 16
; VI-NEXT: s_or_b32 s42, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s23
-; VI-NEXT: s_lshl_b32 s5, s84, 16
+; VI-NEXT: s_lshl_b32 s5, s82, 16
; VI-NEXT: s_or_b32 s43, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s24
-; VI-NEXT: s_lshl_b32 s5, s81, 16
+; VI-NEXT: s_lshl_b32 s5, s71, 16
; VI-NEXT: s_or_b32 s44, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s25
-; VI-NEXT: s_lshl_b32 s5, s70, 16
+; VI-NEXT: s_lshl_b32 s5, s68, 16
; VI-NEXT: s_or_b32 s45, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s26
-; VI-NEXT: s_lshl_b32 s5, s68, 16
+; VI-NEXT: s_lshl_b32 s5, s34, 16
; VI-NEXT: s_or_b32 s46, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s27
-; VI-NEXT: s_lshl_b32 s5, s31, 16
+; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; VI-NEXT: s_or_b32 s47, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s28
-; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_lshl_b32 s5, s77, 16
; VI-NEXT: s_or_b32 s48, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s29
; VI-NEXT: s_lshl_b32 s5, s56, 16
; VI-NEXT: s_or_b32 s49, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s7
-; VI-NEXT: s_lshl_b32 s5, s11, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s86
+; VI-NEXT: s_lshl_b32 s5, s7, 16
; VI-NEXT: s_or_b32 s50, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s85
-; VI-NEXT: s_lshl_b32 s5, s87, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s83
+; VI-NEXT: s_lshl_b32 s5, s85, 16
; VI-NEXT: s_or_b32 s51, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s82
-; VI-NEXT: s_lshl_b32 s5, s83, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s80
+; VI-NEXT: s_lshl_b32 s5, s81, 16
; VI-NEXT: s_or_b32 s52, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s71
-; VI-NEXT: s_lshl_b32 s5, s80, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s69
+; VI-NEXT: s_lshl_b32 s5, s70, 16
; VI-NEXT: s_or_b32 s53, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s5, s69, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s5, s35, 16
; VI-NEXT: s_or_b32 s54, s4, s5
-; VI-NEXT: s_and_b32 s4, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s5, s34, 16
+; VI-NEXT: s_and_b32 s4, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s5, s30, 16
; VI-NEXT: s_or_b32 s55, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s89
-; VI-NEXT: s_lshl_b32 s5, s64, 16
-; VI-NEXT: s_mov_b32 s91, s56
+; VI-NEXT: s_lshl_b32 s5, s88, 16
+; VI-NEXT: s_mov_b32 s90, s56
; VI-NEXT: s_or_b32 s56, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s78
; VI-NEXT: s_lshl_b32 s5, s57, 16
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s88, s57
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s77, s57
; VI-NEXT: s_or_b32 s57, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s76
; VI-NEXT: s_lshl_b32 s5, s58, 16
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s88, s58
; VI-NEXT: s_or_b32 s58, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s74
; VI-NEXT: s_lshl_b32 s5, s59, 16
@@ -45021,152 +45830,158 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; VI-NEXT: s_lshl_b32 s5, s63, 16
; VI-NEXT: s_or_b32 s63, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s8
-; VI-NEXT: s_lshl_b32 s5, vcc_hi, 16
-; VI-NEXT: s_mov_b32 s90, s64
+; VI-NEXT: s_lshl_b32 s5, s75, 16
; VI-NEXT: s_or_b32 s64, s4, s5
; VI-NEXT: s_and_b32 s4, 0xffff, s6
-; VI-NEXT: s_lshl_b32 s5, vcc_lo, 16
+; VI-NEXT: s_lshl_b32 s5, s65, 16
+; VI-NEXT: s_mov_b32 s75, s65
; VI-NEXT: s_or_b32 s65, s4, s5
-; VI-NEXT: s_cbranch_execnz .LBB55_4
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b32 s79, s88
+; VI-NEXT: s_mov_b32 s91, s77
+; VI-NEXT: s_mov_b32 s90, s56
+; VI-NEXT: s_mov_b32 s77, s57
+; VI-NEXT: s_mov_b32 s88, s58
+; VI-NEXT: s_mov_b32 s75, s65
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v29, 0x200
-; VI-NEXT: v_mov_b32_e32 v0, s77
-; VI-NEXT: v_mov_b32_e32 v2, s75
+; VI-NEXT: v_mov_b32_e32 v0, s73
+; VI-NEXT: v_mov_b32_e32 v2, s15
; VI-NEXT: v_add_f16_sdwa v0, v0, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v1, s16, v29
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s17, v29
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_or_b32_e32 v1, v3, v2
-; VI-NEXT: v_mov_b32_e32 v2, s73
+; VI-NEXT: v_mov_b32_e32 v2, s13
; VI-NEXT: v_add_f16_sdwa v2, v2, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v3, s18, v29
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v3, s11
; VI-NEXT: v_add_f16_sdwa v3, v3, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v4, s19, v29
; VI-NEXT: v_or_b32_e32 v3, v4, v3
-; VI-NEXT: v_mov_b32_e32 v4, s13
+; VI-NEXT: v_mov_b32_e32 v4, s9
; VI-NEXT: v_add_f16_sdwa v4, v4, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v5, s20, v29
; VI-NEXT: v_or_b32_e32 v4, v5, v4
-; VI-NEXT: v_mov_b32_e32 v5, s9
+; VI-NEXT: v_mov_b32_e32 v5, s87
; VI-NEXT: v_add_f16_sdwa v5, v5, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v6, s21, v29
; VI-NEXT: v_or_b32_e32 v5, v6, v5
-; VI-NEXT: v_mov_b32_e32 v6, s86
+; VI-NEXT: v_mov_b32_e32 v6, s84
; VI-NEXT: v_add_f16_sdwa v6, v6, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v7, s22, v29
; VI-NEXT: v_or_b32_e32 v6, v7, v6
-; VI-NEXT: v_mov_b32_e32 v7, s84
+; VI-NEXT: v_mov_b32_e32 v7, s82
; VI-NEXT: v_add_f16_sdwa v7, v7, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v8, s23, v29
; VI-NEXT: v_or_b32_e32 v7, v8, v7
-; VI-NEXT: v_mov_b32_e32 v8, s81
+; VI-NEXT: v_mov_b32_e32 v8, s71
; VI-NEXT: v_add_f16_sdwa v8, v8, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v9, s24, v29
; VI-NEXT: v_or_b32_e32 v8, v9, v8
-; VI-NEXT: v_mov_b32_e32 v9, s70
+; VI-NEXT: v_mov_b32_e32 v9, s68
; VI-NEXT: v_add_f16_sdwa v9, v9, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v10, s25, v29
; VI-NEXT: v_or_b32_e32 v9, v10, v9
-; VI-NEXT: v_mov_b32_e32 v10, s68
+; VI-NEXT: v_mov_b32_e32 v10, s34
; VI-NEXT: v_add_f16_sdwa v10, v10, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v11, s26, v29
; VI-NEXT: v_or_b32_e32 v10, v11, v10
-; VI-NEXT: v_mov_b32_e32 v11, s31
+; VI-NEXT: v_mov_b32_e32 v11, vcc_hi
; VI-NEXT: v_add_f16_sdwa v11, v11, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v12, s27, v29
; VI-NEXT: v_or_b32_e32 v11, v12, v11
-; VI-NEXT: v_mov_b32_e32 v12, s79
+; VI-NEXT: v_mov_b32_e32 v12, s91
; VI-NEXT: v_add_f16_sdwa v12, v12, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v13, s28, v29
; VI-NEXT: v_or_b32_e32 v12, v13, v12
-; VI-NEXT: v_mov_b32_e32 v13, s91
+; VI-NEXT: v_mov_b32_e32 v13, s90
; VI-NEXT: v_add_f16_sdwa v13, v13, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v14, s29, v29
; VI-NEXT: v_or_b32_e32 v13, v14, v13
-; VI-NEXT: v_mov_b32_e32 v14, s11
+; VI-NEXT: v_mov_b32_e32 v14, s7
; VI-NEXT: v_add_f16_sdwa v14, v14, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v15, s7, v29
+; VI-NEXT: v_add_f16_e32 v15, s86, v29
; VI-NEXT: v_or_b32_e32 v14, v15, v14
-; VI-NEXT: v_mov_b32_e32 v15, s87
+; VI-NEXT: v_mov_b32_e32 v15, s85
; VI-NEXT: v_add_f16_sdwa v15, v15, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v16, s85, v29
+; VI-NEXT: v_add_f16_e32 v16, s83, v29
; VI-NEXT: v_or_b32_e32 v15, v16, v15
-; VI-NEXT: v_mov_b32_e32 v16, s83
+; VI-NEXT: v_mov_b32_e32 v16, s81
; VI-NEXT: v_add_f16_sdwa v16, v16, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v17, s82, v29
+; VI-NEXT: v_add_f16_e32 v17, s80, v29
; VI-NEXT: v_or_b32_e32 v16, v17, v16
-; VI-NEXT: v_mov_b32_e32 v17, s80
+; VI-NEXT: v_mov_b32_e32 v17, s70
; VI-NEXT: v_add_f16_sdwa v17, v17, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v18, s71, v29
+; VI-NEXT: v_add_f16_e32 v18, s69, v29
; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: v_mov_b32_e32 v18, s69
+; VI-NEXT: v_mov_b32_e32 v18, s35
; VI-NEXT: v_add_f16_sdwa v18, v18, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v19, s35, v29
+; VI-NEXT: v_add_f16_e32 v19, s31, v29
; VI-NEXT: v_or_b32_e32 v18, v19, v18
-; VI-NEXT: v_mov_b32_e32 v19, s34
+; VI-NEXT: v_mov_b32_e32 v19, s30
; VI-NEXT: v_add_f16_sdwa v19, v19, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_add_f16_e32 v20, s30, v29
+; VI-NEXT: v_add_f16_e32 v20, vcc_lo, v29
; VI-NEXT: v_or_b32_e32 v19, v20, v19
-; VI-NEXT: v_mov_b32_e32 v20, s90
+; VI-NEXT: v_mov_b32_e32 v20, s79
; VI-NEXT: v_add_f16_sdwa v20, v20, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, s89, v29
; VI-NEXT: v_or_b32_e32 v20, v21, v20
-; VI-NEXT: v_mov_b32_e32 v21, s88
+; VI-NEXT: v_mov_b32_e32 v21, s77
; VI-NEXT: v_add_f16_sdwa v21, v21, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, s78, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 7
; VI-NEXT: v_or_b32_e32 v21, v22, v21
-; VI-NEXT: v_mov_b32_e32 v22, s4
+; VI-NEXT: v_mov_b32_e32 v22, s88
; VI-NEXT: v_add_f16_sdwa v22, v22, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v23, s76, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 6
+; VI-NEXT: v_readlane_b32 s4, v33, 5
; VI-NEXT: v_or_b32_e32 v22, v23, v22
; VI-NEXT: v_mov_b32_e32 v23, s4
; VI-NEXT: v_add_f16_sdwa v23, v23, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v24, s74, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 5
+; VI-NEXT: v_readlane_b32 s4, v33, 4
; VI-NEXT: v_or_b32_e32 v23, v24, v23
; VI-NEXT: v_mov_b32_e32 v24, s4
; VI-NEXT: v_add_f16_sdwa v24, v24, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v25, s72, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 4
+; VI-NEXT: v_readlane_b32 s4, v33, 3
; VI-NEXT: v_or_b32_e32 v24, v25, v24
; VI-NEXT: v_mov_b32_e32 v25, s4
; VI-NEXT: v_add_f16_sdwa v25, v25, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, s14, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 3
+; VI-NEXT: v_readlane_b32 s4, v33, 2
; VI-NEXT: v_or_b32_e32 v25, v26, v25
; VI-NEXT: v_mov_b32_e32 v26, s4
; VI-NEXT: v_add_f16_sdwa v26, v26, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v27, s12, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 2
+; VI-NEXT: v_readlane_b32 s4, v33, 1
; VI-NEXT: v_or_b32_e32 v26, v27, v26
; VI-NEXT: v_mov_b32_e32 v27, s4
; VI-NEXT: v_add_f16_sdwa v27, v27, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v28, s10, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 1
+; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v27, v28, v27
; VI-NEXT: v_mov_b32_e32 v28, s4
; VI-NEXT: v_add_f16_sdwa v28, v28, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v30, s8, v29
-; VI-NEXT: v_readlane_b32 s4, v33, 0
; VI-NEXT: v_or_b32_e32 v28, v30, v28
-; VI-NEXT: v_mov_b32_e32 v30, s4
+; VI-NEXT: v_mov_b32_e32 v30, s75
; VI-NEXT: v_add_f16_sdwa v30, v30, v29 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v29, s6, v29
; VI-NEXT: v_or_b32_e32 v29, v29, v30
-; VI-NEXT: s_branch .LBB55_5
-; VI-NEXT: .LBB55_3:
-; VI-NEXT: s_mov_b32 s90, s64
-; VI-NEXT: s_mov_b32 s79, s88
-; VI-NEXT: s_mov_b32 s91, s56
-; VI-NEXT: s_mov_b32 s88, s57
-; VI-NEXT: ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
-; VI-NEXT: s_branch .LBB55_2
-; VI-NEXT: .LBB55_4:
+; VI-NEXT: s_branch .LBB55_6
+; VI-NEXT: .LBB55_5:
; VI-NEXT: v_mov_b32_e32 v0, s36
; VI-NEXT: v_mov_b32_e32 v1, s37
; VI-NEXT: v_mov_b32_e32 v2, s38
@@ -45199,7 +46014,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: v_mov_b32_e32 v30, s66
; VI-NEXT: v_mov_b32_e32 v31, s67
-; VI-NEXT: .LBB55_5: ; %end
+; VI-NEXT: .LBB55_6: ; %end
; VI-NEXT: v_readlane_b32 s30, v32, 30
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -45337,10 +46152,18 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; GFX9-NEXT: s_pack_ll_b32_b16 s63, s63, s76
; GFX9-NEXT: s_pack_ll_b32_b16 s64, s74, s75
; GFX9-NEXT: s_pack_ll_b32_b16 s65, s72, s73
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v29, 0x200
; GFX9-NEXT: v_pk_add_f16 v0, s36, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s37, v29 op_sel_hi:[1,0]
@@ -45372,10 +46195,8 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; GFX9-NEXT: v_pk_add_f16 v27, s63, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v28, s64, v29 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v29, s65, v29 op_sel_hi:[1,0]
-; GFX9-NEXT: s_branch .LBB55_5
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: s_branch .LBB55_6
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s36
; GFX9-NEXT: v_mov_b32_e32 v1, s37
; GFX9-NEXT: v_mov_b32_e32 v2, s38
@@ -45408,7 +46229,7 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v29, s65
; GFX9-NEXT: v_mov_b32_e32 v30, s66
; GFX9-NEXT: v_mov_b32_e32 v31, s67
-; GFX9-NEXT: .LBB55_5: ; %end
+; GFX9-NEXT: .LBB55_6: ; %end
; GFX9-NEXT: v_readlane_b32 s65, v32, 13
; GFX9-NEXT: v_readlane_b32 s64, v32, 12
; GFX9-NEXT: v_readlane_b32 s55, v32, 11
@@ -45507,11 +46328,16 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; GFX11-NEXT: s_pack_ll_b32_b16 s27, s56, s61
; GFX11-NEXT: s_pack_ll_b32_b16 s28, s46, s59
; GFX11-NEXT: s_pack_ll_b32_b16 s29, s45, s58
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s40
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s40, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s40, s40, exec_lo
+; GFX11-NEXT: s_cselect_b32 s40, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s40, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
@@ -45543,8 +46369,6 @@ define inreg <15 x double> @bitcast_v60f16_to_v15f64_scalar(<60 x half> inreg %a
; GFX11-NEXT: v_pk_add_f16 v28, 0x200, s28 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v29, 0x200, s29 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -47098,227 +47922,271 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; SI-NEXT: v_writelane_b32 v30, s99, 33
; SI-NEXT: v_writelane_b32 v30, s30, 34
; SI-NEXT: v_writelane_b32 v30, s31, 35
+; SI-NEXT: s_lshr_b32 s5, s24, 16
; SI-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; SI-NEXT: s_mov_b32 s35, s17
+; SI-NEXT: s_lshr_b32 s6, s26, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v31, s18, 0
-; SI-NEXT: v_writelane_b32 v31, s20, 1
+; SI-NEXT: v_writelane_b32 v31, s5, 0
+; SI-NEXT: s_lshr_b32 s7, s28, 16
+; SI-NEXT: v_readfirstlane_b32 s99, v0
+; SI-NEXT: v_writelane_b32 v31, s6, 1
+; SI-NEXT: v_readfirstlane_b32 s98, v2
+; SI-NEXT: s_lshr_b32 s8, s99, 16
+; SI-NEXT: v_writelane_b32 v31, s7, 2
+; SI-NEXT: v_readfirstlane_b32 s64, v4
+; SI-NEXT: s_lshr_b32 s9, s98, 16
+; SI-NEXT: v_writelane_b32 v31, s8, 3
+; SI-NEXT: v_readfirstlane_b32 s66, v6
+; SI-NEXT: s_lshr_b32 s10, s64, 16
+; SI-NEXT: v_writelane_b32 v31, s9, 4
+; SI-NEXT: v_readfirstlane_b32 s68, v8
+; SI-NEXT: s_lshr_b32 s11, s66, 16
+; SI-NEXT: v_writelane_b32 v31, s10, 5
+; SI-NEXT: s_mov_b32 s35, s17
+; SI-NEXT: s_lshr_b32 s79, s17, 16
+; SI-NEXT: v_readfirstlane_b32 s17, v10
+; SI-NEXT: s_lshr_b32 s13, s68, 16
+; SI-NEXT: v_writelane_b32 v31, s11, 6
; SI-NEXT: s_mov_b32 s37, s16
-; SI-NEXT: s_lshr_b32 s78, s17, 16
-; SI-NEXT: s_lshr_b32 s48, s16, 16
+; SI-NEXT: s_lshr_b32 s39, s16, 16
+; SI-NEXT: v_readfirstlane_b32 s16, v12
+; SI-NEXT: s_lshr_b32 s15, s17, 16
+; SI-NEXT: v_writelane_b32 v31, s13, 7
; SI-NEXT: v_readfirstlane_b32 s74, v15
; SI-NEXT: v_readfirstlane_b32 s75, v14
; SI-NEXT: v_readfirstlane_b32 s70, v13
-; SI-NEXT: v_readfirstlane_b32 s16, v12
+; SI-NEXT: s_lshr_b32 s41, s16, 16
; SI-NEXT: v_readfirstlane_b32 s69, v11
-; SI-NEXT: v_readfirstlane_b32 s17, v10
; SI-NEXT: v_readfirstlane_b32 s67, v9
-; SI-NEXT: v_readfirstlane_b32 s68, v8
; SI-NEXT: v_readfirstlane_b32 s65, v7
-; SI-NEXT: v_readfirstlane_b32 s66, v6
; SI-NEXT: v_readfirstlane_b32 s96, v5
-; SI-NEXT: v_readfirstlane_b32 s64, v4
-; SI-NEXT: v_readfirstlane_b32 s97, v3
-; SI-NEXT: v_readfirstlane_b32 s87, v2
-; SI-NEXT: v_readfirstlane_b32 s51, v1
-; SI-NEXT: v_readfirstlane_b32 s99, v0
-; SI-NEXT: v_writelane_b32 v31, s22, 2
+; SI-NEXT: v_readfirstlane_b32 s87, v3
+; SI-NEXT: v_readfirstlane_b32 s97, v1
+; SI-NEXT: v_writelane_b32 v31, s15, 8
; SI-NEXT: s_lshr_b32 s31, s29, 16
-; SI-NEXT: s_lshr_b32 s76, s28, 16
-; SI-NEXT: s_lshr_b32 s30, s27, 16
-; SI-NEXT: s_lshr_b32 s84, s26, 16
-; SI-NEXT: s_lshr_b32 s94, s25, 16
-; SI-NEXT: s_lshr_b32 s55, s24, 16
-; SI-NEXT: s_lshr_b32 s92, s23, 16
-; SI-NEXT: s_lshr_b32 s54, s22, 16
-; SI-NEXT: s_lshr_b32 s90, s21, 16
-; SI-NEXT: s_lshr_b32 s52, s20, 16
-; SI-NEXT: s_lshr_b32 s88, s19, 16
-; SI-NEXT: s_lshr_b32 s50, s18, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s27, 16
+; SI-NEXT: s_lshr_b32 s95, s25, 16
+; SI-NEXT: s_lshr_b32 s93, s23, 16
+; SI-NEXT: s_lshr_b32 s53, s22, 16
+; SI-NEXT: s_lshr_b32 s91, s21, 16
+; SI-NEXT: s_lshr_b32 s51, s20, 16
+; SI-NEXT: s_lshr_b32 s89, s19, 16
+; SI-NEXT: s_lshr_b32 s49, s18, 16
; SI-NEXT: s_lshr_b32 s86, s74, 16
-; SI-NEXT: s_lshr_b32 s93, s75, 16
+; SI-NEXT: s_lshr_b32 s77, s75, 16
; SI-NEXT: s_lshr_b32 s85, s70, 16
-; SI-NEXT: s_lshr_b32 s91, s16, 16
-; SI-NEXT: s_lshr_b32 s39, s69, 16
-; SI-NEXT: s_lshr_b32 s89, s17, 16
-; SI-NEXT: s_lshr_b32 s83, s67, 16
-; SI-NEXT: s_lshr_b32 s79, s68, 16
-; SI-NEXT: s_lshr_b32 s82, s65, 16
-; SI-NEXT: s_lshr_b32 s77, s66, 16
-; SI-NEXT: s_lshr_b32 s81, s96, 16
-; SI-NEXT: s_lshr_b32 s98, s64, 16
-; SI-NEXT: s_lshr_b32 s80, s97, 16
-; SI-NEXT: s_lshr_b32 s95, s87, 16
-; SI-NEXT: s_lshr_b32 s53, s51, 16
-; SI-NEXT: s_lshr_b32 s71, s99, 16
+; SI-NEXT: s_lshr_b32 s84, s69, 16
+; SI-NEXT: s_lshr_b32 s71, s67, 16
+; SI-NEXT: s_lshr_b32 s90, s65, 16
+; SI-NEXT: s_lshr_b32 s88, s96, 16
+; SI-NEXT: s_lshr_b32 s78, s87, 16
+; SI-NEXT: s_lshr_b32 s83, s97, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
-; SI-NEXT: v_writelane_b32 v31, s24, 3
+; SI-NEXT: v_writelane_b32 v31, s41, 9
+; SI-NEXT: s_mov_b32 s81, s22
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: v_writelane_b32 v31, s26, 4
-; SI-NEXT: v_writelane_b32 v31, s28, 5
-; SI-NEXT: s_cbranch_scc0 .LBB57_4
+; SI-NEXT: v_writelane_b32 v31, s77, 10
+; SI-NEXT: v_writelane_b32 v31, s24, 11
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
+; SI-NEXT: s_lshl_b32 s56, s5, 16
+; SI-NEXT: s_lshl_b32 s44, s7, 16
; SI-NEXT: s_and_b32 s5, s35, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s78, 16
+; SI-NEXT: s_lshl_b32 s7, s79, 16
; SI-NEXT: s_or_b32 s73, s5, s7
; SI-NEXT: s_and_b32 s5, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s88, 16
+; SI-NEXT: s_lshl_b32 s7, s89, 16
; SI-NEXT: s_or_b32 s63, s5, s7
; SI-NEXT: s_and_b32 s5, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s90, 16
+; SI-NEXT: s_lshl_b32 s7, s91, 16
; SI-NEXT: s_or_b32 s61, s5, s7
; SI-NEXT: s_and_b32 s5, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s92, 16
+; SI-NEXT: s_lshl_b32 s7, s93, 16
; SI-NEXT: s_or_b32 s59, s5, s7
; SI-NEXT: s_and_b32 s5, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s94, 16
+; SI-NEXT: s_lshl_b32 s7, s95, 16
; SI-NEXT: s_or_b32 s57, s5, s7
; SI-NEXT: s_and_b32 s5, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s30, 16
+; SI-NEXT: s_lshl_b32 s7, vcc_hi, 16
; SI-NEXT: s_or_b32 s47, s5, s7
; SI-NEXT: s_and_b32 s5, s29, 0xffff
; SI-NEXT: s_lshl_b32 s7, s31, 16
; SI-NEXT: s_or_b32 s45, s5, s7
-; SI-NEXT: s_and_b32 s5, s51, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s53, 16
-; SI-NEXT: s_or_b32 s43, s5, s7
; SI-NEXT: s_and_b32 s5, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s80, 16
+; SI-NEXT: s_lshl_b32 s7, s83, 16
+; SI-NEXT: s_or_b32 s43, s5, s7
+; SI-NEXT: s_and_b32 s5, s87, 0xffff
+; SI-NEXT: s_lshl_b32 s7, s78, 16
+; SI-NEXT: s_lshl_b32 s46, s6, 16
+; SI-NEXT: s_lshl_b32 s6, s41, 16
; SI-NEXT: s_or_b32 s41, s5, s7
; SI-NEXT: s_and_b32 s5, s96, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s81, 16
+; SI-NEXT: s_lshl_b32 s7, s88, 16
+; SI-NEXT: s_lshl_b32 s42, s8, 16
+; SI-NEXT: s_lshl_b32 s8, s15, 16
; SI-NEXT: s_or_b32 s15, s5, s7
; SI-NEXT: s_and_b32 s5, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s82, 16
+; SI-NEXT: s_lshl_b32 s7, s90, 16
+; SI-NEXT: s_lshl_b32 s14, s10, 16
+; SI-NEXT: s_lshl_b32 s10, s13, 16
; SI-NEXT: s_or_b32 s13, s5, s7
; SI-NEXT: s_and_b32 s5, s67, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s83, 16
+; SI-NEXT: s_lshl_b32 s7, s71, 16
+; SI-NEXT: s_lshl_b32 s12, s11, 16
; SI-NEXT: s_or_b32 s11, s5, s7
; SI-NEXT: s_and_b32 s5, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s7, s39, 16
+; SI-NEXT: s_lshl_b32 s7, s84, 16
+; SI-NEXT: s_and_b32 s4, s37, 0xffff
+; SI-NEXT: s_lshl_b32 s72, s39, 16
+; SI-NEXT: s_lshl_b32 s40, s9, 16
; SI-NEXT: s_or_b32 s9, s5, s7
; SI-NEXT: s_and_b32 s5, s70, 0xffff
; SI-NEXT: s_lshl_b32 s7, s85, 16
-; SI-NEXT: s_and_b32 s4, s37, 0xffff
-; SI-NEXT: s_lshl_b32 s72, s48, 16
-; SI-NEXT: s_lshl_b32 s44, s76, 16
-; SI-NEXT: s_lshl_b32 s42, s71, 16
+; SI-NEXT: s_or_b32 s4, s4, s72
+; SI-NEXT: s_lshl_b32 s62, s49, 16
; SI-NEXT: s_or_b32 s7, s5, s7
; SI-NEXT: s_and_b32 s5, s74, 0xffff
-; SI-NEXT: s_mov_b32 s49, s71
-; SI-NEXT: s_mov_b32 s71, s76
; SI-NEXT: s_lshl_b32 s76, s86, 16
-; SI-NEXT: s_or_b32 s4, s4, s72
-; SI-NEXT: s_or_b32 vcc_hi, s5, s76
-; SI-NEXT: s_mov_b32 s5, s73
-; SI-NEXT: s_lshr_b64 s[72:73], s[72:73], 16
-; SI-NEXT: v_writelane_b32 v31, s72, 6
-; SI-NEXT: s_lshl_b32 s62, s50, 16
-; SI-NEXT: v_writelane_b32 v31, s73, 7
+; SI-NEXT: s_mov_b32 s22, s81
+; SI-NEXT: s_lshr_b64 s[80:81], s[72:73], 16
; SI-NEXT: s_and_b32 s72, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s60, s52, 16
+; SI-NEXT: s_lshl_b32 s60, s51, 16
+; SI-NEXT: s_lshl_b32 s58, s53, 16
+; SI-NEXT: s_mov_b32 s54, s53
+; SI-NEXT: s_or_b32 s53, s5, s76
+; SI-NEXT: s_lshl_b32 s52, s77, 16
+; SI-NEXT: s_mov_b32 s81, s22
; SI-NEXT: s_or_b32 s72, s72, s62
-; SI-NEXT: s_mov_b32 s76, s19
-; SI-NEXT: s_lshr_b64 s[18:19], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[62:63], 16
; SI-NEXT: s_and_b32 s62, s20, 0xffff
-; SI-NEXT: s_mov_b32 s73, s63
; SI-NEXT: s_or_b32 s62, s62, s60
-; SI-NEXT: s_mov_b32 s63, s61
-; SI-NEXT: s_lshr_b64 s[60:61], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v31, s60, 8
-; SI-NEXT: s_lshl_b32 s58, s54, 16
-; SI-NEXT: v_writelane_b32 v31, s61, 9
-; SI-NEXT: s_and_b32 s60, s22, 0xffff
+; SI-NEXT: s_mov_b32 s22, s21
+; SI-NEXT: s_mov_b32 s21, s83
+; SI-NEXT: s_lshr_b64 s[82:83], s[60:61], 16
+; SI-NEXT: s_and_b32 s60, s81, 0xffff
+; SI-NEXT: s_mov_b32 s83, s21
+; SI-NEXT: s_mov_b32 s21, s22
; SI-NEXT: s_or_b32 s60, s60, s58
-; SI-NEXT: s_mov_b32 s61, s59
-; SI-NEXT: s_lshr_b64 s[58:59], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v31, s58, 10
-; SI-NEXT: s_lshl_b32 s56, s55, 16
-; SI-NEXT: v_writelane_b32 v31, s59, 11
+; SI-NEXT: s_mov_b32 s77, s89
+; SI-NEXT: s_mov_b32 s89, s23
+; SI-NEXT: s_lshr_b64 s[22:23], s[58:59], 16
; SI-NEXT: s_and_b32 s58, s24, 0xffff
+; SI-NEXT: s_mov_b32 s23, s89
+; SI-NEXT: s_mov_b32 s89, s77
; SI-NEXT: s_or_b32 s58, s58, s56
-; SI-NEXT: s_mov_b32 s59, s57
-; SI-NEXT: s_lshr_b64 s[56:57], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v31, s56, 12
-; SI-NEXT: s_lshl_b32 s46, s84, 16
-; SI-NEXT: v_writelane_b32 v31, s57, 13
+; SI-NEXT: s_mov_b32 s77, s91
+; SI-NEXT: s_mov_b32 s91, s25
+; SI-NEXT: s_lshr_b64 s[24:25], s[56:57], 16
; SI-NEXT: s_and_b32 s56, s26, 0xffff
+; SI-NEXT: s_mov_b32 s25, s91
+; SI-NEXT: s_mov_b32 s91, s77
; SI-NEXT: s_or_b32 s56, s56, s46
-; SI-NEXT: s_mov_b32 s57, s47
-; SI-NEXT: s_lshr_b64 s[46:47], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v31, s46, 14
-; SI-NEXT: v_writelane_b32 v31, s47, 15
+; SI-NEXT: s_mov_b32 s77, s93
+; SI-NEXT: s_lshr_b64 s[92:93], s[46:47], 16
; SI-NEXT: s_and_b32 s46, s28, 0xffff
+; SI-NEXT: s_mov_b32 s93, s77
; SI-NEXT: s_or_b32 s46, s46, s44
-; SI-NEXT: s_mov_b32 s47, s45
-; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v31, s44, 16
-; SI-NEXT: v_writelane_b32 v31, s45, 17
+; SI-NEXT: s_mov_b32 s77, s95
+; SI-NEXT: s_lshr_b64 s[94:95], s[44:45], 16
; SI-NEXT: s_and_b32 s44, s99, 0xffff
+; SI-NEXT: s_mov_b32 s95, s77
; SI-NEXT: s_or_b32 s44, s44, s42
-; SI-NEXT: s_mov_b32 s45, s43
-; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 16
-; SI-NEXT: v_writelane_b32 v31, s42, 18
-; SI-NEXT: s_lshl_b32 s40, s95, 16
-; SI-NEXT: v_writelane_b32 v31, s43, 19
-; SI-NEXT: s_and_b32 s42, s87, 0xffff
-; SI-NEXT: s_lshl_b32 s14, s98, 16
+; SI-NEXT: s_mov_b32 s77, vcc_hi
+; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
+; SI-NEXT: s_and_b32 s42, s98, 0xffff
+; SI-NEXT: s_mov_b32 vcc_hi, s77
; SI-NEXT: s_or_b32 s42, s42, s40
-; SI-NEXT: s_mov_b32 s36, s35
-; SI-NEXT: s_lshr_b64 s[34:35], s[40:41], 16
+; SI-NEXT: s_mov_b32 s77, s31
+; SI-NEXT: s_lshr_b64 s[30:31], s[40:41], 16
; SI-NEXT: s_and_b32 s40, s64, 0xffff
-; SI-NEXT: s_lshl_b32 s12, s77, 16
-; SI-NEXT: s_mov_b32 s35, s36
+; SI-NEXT: s_mov_b32 s31, s77
; SI-NEXT: s_or_b32 s40, s40, s14
-; SI-NEXT: s_mov_b32 s38, s37
-; SI-NEXT: s_lshr_b64 s[36:37], s[14:15], 16
+; SI-NEXT: s_mov_b32 s77, s35
+; SI-NEXT: s_lshr_b64 s[34:35], s[14:15], 16
; SI-NEXT: s_and_b32 s14, s66, 0xffff
-; SI-NEXT: s_mov_b32 s43, s41
-; SI-NEXT: s_mov_b32 s41, s15
+; SI-NEXT: s_mov_b32 s35, s77
; SI-NEXT: s_or_b32 s14, s14, s12
-; SI-NEXT: s_mov_b32 s15, s13
-; SI-NEXT: s_lshr_b64 s[12:13], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v31, s12, 20
-; SI-NEXT: s_lshl_b32 s10, s79, 16
-; SI-NEXT: v_writelane_b32 v31, s13, 21
+; SI-NEXT: s_mov_b32 s77, s37
+; SI-NEXT: s_lshr_b64 s[36:37], s[12:13], 16
; SI-NEXT: s_and_b32 s12, s68, 0xffff
+; SI-NEXT: s_mov_b32 s37, s77
; SI-NEXT: s_or_b32 s12, s12, s10
-; SI-NEXT: s_mov_b32 s13, s11
-; SI-NEXT: s_lshr_b64 s[10:11], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v31, s10, 22
-; SI-NEXT: s_lshl_b32 s8, s89, 16
-; SI-NEXT: v_writelane_b32 v31, s11, 23
+; SI-NEXT: s_mov_b32 s77, s39
+; SI-NEXT: s_lshr_b64 s[38:39], s[10:11], 16
; SI-NEXT: s_and_b32 s10, s17, 0xffff
+; SI-NEXT: s_mov_b32 s39, s77
; SI-NEXT: s_or_b32 s10, s10, s8
-; SI-NEXT: s_mov_b32 s11, s9
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 16
-; SI-NEXT: v_writelane_b32 v31, s8, 24
-; SI-NEXT: s_lshl_b32 s6, s91, 16
-; SI-NEXT: v_writelane_b32 v31, s9, 25
+; SI-NEXT: s_mov_b32 s77, s49
+; SI-NEXT: s_lshr_b64 s[48:49], s[8:9], 16
; SI-NEXT: s_and_b32 s8, s16, 0xffff
+; SI-NEXT: s_mov_b32 s49, s77
; SI-NEXT: s_or_b32 s8, s8, s6
-; SI-NEXT: s_mov_b32 s9, s7
-; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
-; SI-NEXT: s_lshl_b32 vcc_lo, s93, 16
-; SI-NEXT: v_writelane_b32 v31, s6, 26
-; SI-NEXT: s_mov_b32 s19, s76
-; SI-NEXT: s_mov_b32 s37, s38
-; SI-NEXT: s_mov_b32 s76, s84
-; SI-NEXT: v_writelane_b32 v31, s7, 27
+; SI-NEXT: s_mov_b32 s77, s51
+; SI-NEXT: s_lshr_b64 s[50:51], s[6:7], 16
; SI-NEXT: s_and_b32 s6, s75, 0xffff
-; SI-NEXT: s_mov_b32 s84, s39
-; SI-NEXT: s_lshr_b64 s[38:39], vcc, 16
-; SI-NEXT: s_or_b32 s6, s6, vcc_lo
-; SI-NEXT: s_mov_b32 s7, vcc_hi
-; SI-NEXT: s_mov_b32 s39, s84
-; SI-NEXT: s_mov_b32 s84, s76
-; SI-NEXT: s_mov_b32 s76, s71
-; SI-NEXT: s_mov_b32 s71, s49
-; SI-NEXT: s_cbranch_execnz .LBB57_3
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b32 s5, s73
+; SI-NEXT: s_mov_b32 s73, s63
+; SI-NEXT: s_mov_b32 s63, s61
+; SI-NEXT: s_mov_b32 s61, s59
+; SI-NEXT: s_mov_b32 s59, s57
+; SI-NEXT: s_mov_b32 s57, s47
+; SI-NEXT: s_mov_b32 s47, s45
+; SI-NEXT: s_mov_b32 s45, s43
+; SI-NEXT: s_mov_b32 s43, s41
+; SI-NEXT: s_mov_b32 s41, s15
+; SI-NEXT: s_mov_b32 s15, s13
+; SI-NEXT: s_mov_b32 s13, s11
+; SI-NEXT: s_mov_b32 s11, s9
+; SI-NEXT: s_mov_b32 s9, s7
+; SI-NEXT: s_or_b32 s6, s6, s52
+; SI-NEXT: s_mov_b32 s7, s53
+; SI-NEXT: s_lshr_b64 s[52:53], s[52:53], 16
+; SI-NEXT: s_mov_b32 s51, s77
+; SI-NEXT: s_mov_b32 s53, s54
+; SI-NEXT: s_mov_b64 s[54:55], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[54:55], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr80
+; SI-NEXT: ; implicit-def: $sgpr72
+; SI-NEXT: ; implicit-def: $sgpr76
+; SI-NEXT: ; implicit-def: $sgpr62
+; SI-NEXT: ; implicit-def: $sgpr82
+; SI-NEXT: ; implicit-def: $sgpr60
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr58
+; SI-NEXT: ; implicit-def: $sgpr24
+; SI-NEXT: ; implicit-def: $sgpr56
+; SI-NEXT: ; implicit-def: $sgpr92
+; SI-NEXT: ; implicit-def: $sgpr46
+; SI-NEXT: ; implicit-def: $sgpr94
+; SI-NEXT: ; implicit-def: $sgpr44
+; SI-NEXT: ; implicit-def: $vcc_lo
+; SI-NEXT: ; implicit-def: $sgpr42
+; SI-NEXT: ; implicit-def: $sgpr30
+; SI-NEXT: ; implicit-def: $sgpr40
+; SI-NEXT: ; implicit-def: $sgpr34
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr36
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr38
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr48
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr50
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr52
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[54:55], s[54:55], exec
+; SI-NEXT: s_cselect_b32 s77, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s77, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s75, s75, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 10
; SI-NEXT: s_and_b32 s4, s75, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s93, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s74, s74, 3
; SI-NEXT: s_add_i32 s6, s4, 0x30000
@@ -47326,9 +48194,10 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s86, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 9
; SI-NEXT: s_add_i32 s7, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s16, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s91, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s70, s70, 3
; SI-NEXT: s_add_i32 s8, s4, 0x30000
@@ -47336,308 +48205,268 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s5, s85, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 8
; SI-NEXT: s_add_i32 s9, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s17, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s89, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s69, s69, 3
; SI-NEXT: s_add_i32 s10, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s69, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s39, 16
+; SI-NEXT: s_lshl_b32 s5, s84, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s68, s68, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 7
; SI-NEXT: s_add_i32 s11, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s68, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s79, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s67, s67, 3
; SI-NEXT: s_add_i32 s12, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s67, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s83, 16
+; SI-NEXT: s_lshl_b32 s5, s71, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s66, s66, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 6
; SI-NEXT: s_add_i32 s13, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s66, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s77, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s65, s65, 3
; SI-NEXT: s_add_i32 s14, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s65, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s82, 16
+; SI-NEXT: s_lshl_b32 s5, s90, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s64, s64, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 5
; SI-NEXT: s_add_i32 s15, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s64, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s98, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s96, s96, 3
; SI-NEXT: s_add_i32 s40, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s96, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s81, 16
+; SI-NEXT: s_lshl_b32 s5, s88, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s98, s87, 3
+; SI-NEXT: s_add_i32 s98, s98, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 4
; SI-NEXT: s_add_i32 s41, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s98, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s95, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s87, s97, 3
+; SI-NEXT: s_add_i32 s87, s87, 3
; SI-NEXT: s_add_i32 s42, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s87, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s80, 16
+; SI-NEXT: s_lshl_b32 s5, s78, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s99, s99, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 3
; SI-NEXT: s_add_i32 s43, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s99, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s71, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_add_i32 s97, s51, 3
+; SI-NEXT: s_add_i32 s97, s97, 3
; SI-NEXT: s_add_i32 s44, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s97, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s53, 16
+; SI-NEXT: s_lshl_b32 s5, s83, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s28, s28, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 2
; SI-NEXT: s_add_i32 s45, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 5
-; SI-NEXT: s_add_i32 s28, s4, 3
; SI-NEXT: s_and_b32 s4, s28, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s76, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s29, s29, 3
; SI-NEXT: s_add_i32 s46, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s29, 0xffff
; SI-NEXT: s_lshl_b32 s5, s31, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s26, s26, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 1
; SI-NEXT: s_add_i32 s47, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 4
-; SI-NEXT: s_add_i32 s26, s4, 3
; SI-NEXT: s_and_b32 s4, s26, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s84, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s27, s27, 3
; SI-NEXT: s_add_i32 s56, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s27, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s30, 16
+; SI-NEXT: s_lshl_b32 s5, vcc_hi, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s57, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 3
+; SI-NEXT: v_readlane_b32 s4, v31, 11
; SI-NEXT: s_add_i32 s24, s4, 3
+; SI-NEXT: v_readlane_b32 s5, v31, 0
; SI-NEXT: s_and_b32 s4, s24, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s55, 16
+; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s25, s25, 3
; SI-NEXT: s_add_i32 s58, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s25, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s94, 16
+; SI-NEXT: s_lshl_b32 s5, s95, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s22, s81, 3
; SI-NEXT: s_add_i32 s59, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 2
-; SI-NEXT: s_add_i32 s22, s4, 3
; SI-NEXT: s_and_b32 s4, s22, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s54, 16
+; SI-NEXT: s_lshl_b32 s5, s53, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s23, s23, 3
; SI-NEXT: s_add_i32 s60, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s23, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s92, 16
+; SI-NEXT: s_lshl_b32 s5, s93, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s20, s20, 3
; SI-NEXT: s_add_i32 s61, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 1
-; SI-NEXT: s_add_i32 s20, s4, 3
; SI-NEXT: s_and_b32 s4, s20, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s52, 16
+; SI-NEXT: s_lshl_b32 s5, s51, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s21, s21, 3
; SI-NEXT: s_add_i32 s62, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s21, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s90, 16
+; SI-NEXT: s_lshl_b32 s5, s91, 16
; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s63, s4, 0x30000
-; SI-NEXT: v_readlane_b32 s4, v31, 0
-; SI-NEXT: s_add_i32 s18, s4, 3
; SI-NEXT: s_and_b32 s4, s18, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s50, 16
+; SI-NEXT: s_lshl_b32 s5, s49, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s19, s19, 3
; SI-NEXT: s_add_i32 s72, s4, 0x30000
; SI-NEXT: s_and_b32 s4, s19, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s88, 16
+; SI-NEXT: s_lshl_b32 s5, s89, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s73, s4, 0x30000
; SI-NEXT: s_add_i32 s4, s37, 3
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s48, 16
+; SI-NEXT: s_lshl_b32 s5, s39, 16
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: s_add_i32 s5, s35, 3
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s78, 16
+; SI-NEXT: s_lshl_b32 s16, s79, 16
; SI-NEXT: s_or_b32 s5, s16, s5
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
-; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 6
-; SI-NEXT: v_writelane_b32 v31, s17, 7
-; SI-NEXT: s_lshr_b64 s[16:17], s[62:63], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 8
-; SI-NEXT: v_writelane_b32 v31, s17, 9
-; SI-NEXT: s_lshr_b64 s[16:17], s[60:61], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 10
-; SI-NEXT: v_writelane_b32 v31, s17, 11
-; SI-NEXT: s_lshr_b64 s[16:17], s[58:59], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 12
-; SI-NEXT: v_writelane_b32 v31, s17, 13
-; SI-NEXT: s_lshr_b64 s[16:17], s[56:57], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 14
-; SI-NEXT: v_writelane_b32 v31, s17, 15
-; SI-NEXT: s_lshr_b64 s[16:17], s[46:47], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 16
-; SI-NEXT: v_writelane_b32 v31, s17, 17
-; SI-NEXT: s_lshr_b64 s[16:17], s[44:45], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 18
-; SI-NEXT: v_writelane_b32 v31, s17, 19
-; SI-NEXT: s_lshr_b64 s[16:17], s[14:15], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 20
-; SI-NEXT: v_writelane_b32 v31, s17, 21
-; SI-NEXT: s_lshr_b64 s[16:17], s[12:13], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 22
-; SI-NEXT: v_writelane_b32 v31, s17, 23
-; SI-NEXT: s_lshr_b64 s[16:17], s[10:11], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 24
-; SI-NEXT: v_writelane_b32 v31, s17, 25
-; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
-; SI-NEXT: s_lshr_b64 s[38:39], s[6:7], 16
-; SI-NEXT: s_lshr_b64 s[18:19], s[72:73], 16
-; SI-NEXT: s_lshr_b64 s[34:35], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[36:37], s[40:41], 16
-; SI-NEXT: v_writelane_b32 v31, s16, 26
-; SI-NEXT: s_lshr_b32 s78, s5, 16
-; SI-NEXT: s_lshr_b32 s88, s73, 16
-; SI-NEXT: s_lshr_b32 s90, s63, 16
-; SI-NEXT: s_lshr_b32 s92, s61, 16
-; SI-NEXT: s_lshr_b32 s94, s59, 16
-; SI-NEXT: s_lshr_b32 s30, s57, 16
+; SI-NEXT: s_lshr_b64 s[82:83], s[62:63], 16
+; SI-NEXT: s_lshr_b64 s[92:93], s[56:57], 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[46:47], 16
+; SI-NEXT: s_lshr_b64 vcc, s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[30:31], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[80:81], s[4:5], 16
+; SI-NEXT: s_lshr_b64 s[76:77], s[72:73], 16
+; SI-NEXT: s_lshr_b64 s[22:23], s[60:61], 16
+; SI-NEXT: s_lshr_b64 s[24:25], s[58:59], 16
+; SI-NEXT: s_lshr_b64 s[34:35], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[36:37], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[38:39], s[12:13], 16
+; SI-NEXT: s_lshr_b64 s[48:49], s[10:11], 16
+; SI-NEXT: s_lshr_b64 s[50:51], s[8:9], 16
+; SI-NEXT: s_lshr_b64 s[52:53], s[6:7], 16
+; SI-NEXT: s_lshr_b32 s79, s5, 16
+; SI-NEXT: s_lshr_b32 s89, s73, 16
+; SI-NEXT: s_lshr_b32 s91, s63, 16
+; SI-NEXT: s_lshr_b32 s93, s61, 16
+; SI-NEXT: s_lshr_b32 s95, s59, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s57, 16
; SI-NEXT: s_lshr_b32 s31, s47, 16
-; SI-NEXT: s_lshr_b32 s53, s45, 16
-; SI-NEXT: s_lshr_b32 s80, s43, 16
-; SI-NEXT: s_lshr_b32 s81, s41, 16
-; SI-NEXT: s_lshr_b32 s82, s15, 16
-; SI-NEXT: s_lshr_b32 s83, s13, 16
-; SI-NEXT: s_lshr_b32 s39, s11, 16
+; SI-NEXT: s_lshr_b32 s83, s45, 16
+; SI-NEXT: s_lshr_b32 s78, s43, 16
+; SI-NEXT: s_lshr_b32 s88, s41, 16
+; SI-NEXT: s_lshr_b32 s90, s15, 16
+; SI-NEXT: s_lshr_b32 s71, s13, 16
+; SI-NEXT: s_lshr_b32 s84, s11, 16
; SI-NEXT: s_lshr_b32 s85, s9, 16
; SI-NEXT: s_lshr_b32 s86, s7, 16
-; SI-NEXT: v_writelane_b32 v31, s17, 27
-; SI-NEXT: .LBB57_3: ; %end
-; SI-NEXT: v_readlane_b32 s16, v31, 6
+; SI-NEXT: .LBB57_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: v_readlane_b32 s17, v31, 7
+; SI-NEXT: s_lshl_b32 s16, s80, 16
; SI-NEXT: s_or_b32 s4, s4, s16
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_lshl_b32 s16, s78, 16
+; SI-NEXT: s_lshl_b32 s16, s79, 16
; SI-NEXT: s_or_b32 s5, s5, s16
; SI-NEXT: s_and_b32 s16, s72, 0xffff
-; SI-NEXT: s_lshl_b32 s17, s18, 16
+; SI-NEXT: s_lshl_b32 s17, s76, 16
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s73, 0xffff
-; SI-NEXT: s_lshl_b32 s18, s88, 16
-; SI-NEXT: v_readlane_b32 s20, v31, 8
+; SI-NEXT: s_lshl_b32 s18, s89, 16
; SI-NEXT: s_or_b32 s17, s17, s18
; SI-NEXT: s_and_b32 s18, s62, 0xffff
-; SI-NEXT: s_lshl_b32 s19, s20, 16
-; SI-NEXT: v_readlane_b32 s21, v31, 9
+; SI-NEXT: s_lshl_b32 s19, s82, 16
; SI-NEXT: s_or_b32 s18, s18, s19
; SI-NEXT: s_and_b32 s19, s63, 0xffff
-; SI-NEXT: s_lshl_b32 s20, s90, 16
-; SI-NEXT: v_readlane_b32 s22, v31, 10
+; SI-NEXT: s_lshl_b32 s20, s91, 16
; SI-NEXT: s_or_b32 s19, s19, s20
; SI-NEXT: s_and_b32 s20, s60, 0xffff
; SI-NEXT: s_lshl_b32 s21, s22, 16
-; SI-NEXT: v_readlane_b32 s23, v31, 11
; SI-NEXT: s_or_b32 s20, s20, s21
; SI-NEXT: s_and_b32 s21, s61, 0xffff
-; SI-NEXT: s_lshl_b32 s22, s92, 16
-; SI-NEXT: v_readlane_b32 s24, v31, 12
+; SI-NEXT: s_lshl_b32 s22, s93, 16
; SI-NEXT: s_or_b32 s21, s21, s22
; SI-NEXT: s_and_b32 s22, s58, 0xffff
; SI-NEXT: s_lshl_b32 s23, s24, 16
-; SI-NEXT: v_readlane_b32 s25, v31, 13
; SI-NEXT: s_or_b32 s22, s22, s23
; SI-NEXT: s_and_b32 s23, s59, 0xffff
-; SI-NEXT: s_lshl_b32 s24, s94, 16
-; SI-NEXT: v_readlane_b32 s26, v31, 14
+; SI-NEXT: s_lshl_b32 s24, s95, 16
; SI-NEXT: s_or_b32 s23, s23, s24
; SI-NEXT: s_and_b32 s24, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s25, s26, 16
-; SI-NEXT: v_readlane_b32 s27, v31, 15
+; SI-NEXT: s_lshl_b32 s25, s92, 16
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s57, 0xffff
-; SI-NEXT: s_lshl_b32 s26, s30, 16
-; SI-NEXT: v_readlane_b32 s28, v31, 16
+; SI-NEXT: s_lshl_b32 s26, vcc_hi, 16
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s28, 16
-; SI-NEXT: v_readlane_b32 s29, v31, 17
+; SI-NEXT: s_lshl_b32 s27, s94, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s47, 0xffff
; SI-NEXT: s_lshl_b32 s28, s31, 16
-; SI-NEXT: v_readlane_b32 s46, v31, 18
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s44, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s46, 16
+; SI-NEXT: s_lshl_b32 s29, vcc_lo, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s45, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s53, 16
+; SI-NEXT: s_lshl_b32 s44, s83, 16
; SI-NEXT: s_or_b32 s29, s29, s44
; SI-NEXT: s_and_b32 s42, s42, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s34, 16
+; SI-NEXT: s_lshl_b32 s44, s30, 16
; SI-NEXT: s_or_b32 s42, s42, s44
; SI-NEXT: s_and_b32 s43, s43, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s80, 16
+; SI-NEXT: s_lshl_b32 s44, s78, 16
; SI-NEXT: s_or_b32 s43, s43, s44
; SI-NEXT: s_and_b32 s40, s40, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s36, 16
+; SI-NEXT: s_lshl_b32 s44, s34, 16
; SI-NEXT: s_or_b32 s40, s40, s44
; SI-NEXT: s_and_b32 s41, s41, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s81, 16
+; SI-NEXT: s_lshl_b32 s44, s88, 16
; SI-NEXT: s_or_b32 s41, s41, s44
-; SI-NEXT: v_readlane_b32 s44, v31, 20
; SI-NEXT: s_and_b32 s14, s14, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readlane_b32 s45, v31, 21
+; SI-NEXT: s_lshl_b32 s44, s36, 16
; SI-NEXT: s_or_b32 s14, s14, s44
; SI-NEXT: s_and_b32 s15, s15, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s82, 16
+; SI-NEXT: s_lshl_b32 s44, s90, 16
; SI-NEXT: s_or_b32 s15, s15, s44
-; SI-NEXT: v_readlane_b32 s44, v31, 22
; SI-NEXT: s_and_b32 s12, s12, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readlane_b32 s45, v31, 23
+; SI-NEXT: s_lshl_b32 s44, s38, 16
; SI-NEXT: s_or_b32 s12, s12, s44
; SI-NEXT: s_and_b32 s13, s13, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s83, 16
+; SI-NEXT: s_lshl_b32 s44, s71, 16
; SI-NEXT: s_or_b32 s13, s13, s44
-; SI-NEXT: v_readlane_b32 s44, v31, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: v_readlane_b32 s45, v31, 25
+; SI-NEXT: s_lshl_b32 s44, s48, 16
; SI-NEXT: s_or_b32 s10, s10, s44
; SI-NEXT: s_and_b32 s11, s11, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s39, 16
+; SI-NEXT: s_lshl_b32 s44, s84, 16
; SI-NEXT: s_or_b32 s11, s11, s44
-; SI-NEXT: v_readlane_b32 s44, v31, 26
; SI-NEXT: s_and_b32 s8, s8, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s44, 16
+; SI-NEXT: s_lshl_b32 s44, s50, 16
; SI-NEXT: s_or_b32 s8, s8, s44
; SI-NEXT: s_and_b32 s9, s9, 0xffff
; SI-NEXT: s_lshl_b32 s44, s85, 16
; SI-NEXT: s_or_b32 s9, s9, s44
; SI-NEXT: s_and_b32 s6, s6, 0xffff
-; SI-NEXT: s_lshl_b32 s44, s38, 16
+; SI-NEXT: s_lshl_b32 s44, s52, 16
; SI-NEXT: s_or_b32 s6, s6, s44
; SI-NEXT: s_and_b32 s7, s7, 0xffff
; SI-NEXT: s_lshl_b32 s44, s86, 16
; SI-NEXT: s_or_b32 s7, s7, s44
; SI-NEXT: v_readlane_b32 s30, v30, 34
-; SI-NEXT: v_readlane_b32 s47, v31, 19
-; SI-NEXT: v_readlane_b32 s45, v31, 27
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s16
@@ -47709,60 +48538,6 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB57_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 6
-; SI-NEXT: v_writelane_b32 v31, s7, 7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr72
-; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $sgpr62
-; SI-NEXT: ; implicit-def: $sgpr60
-; SI-NEXT: ; implicit-def: $sgpr58
-; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $sgpr46
-; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $sgpr42
-; SI-NEXT: ; implicit-def: $sgpr34
-; SI-NEXT: ; implicit-def: $sgpr40
-; SI-NEXT: ; implicit-def: $sgpr36
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr38
-; SI-NEXT: v_writelane_b32 v31, s6, 8
-; SI-NEXT: v_writelane_b32 v31, s7, 9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 10
-; SI-NEXT: v_writelane_b32 v31, s7, 11
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 12
-; SI-NEXT: v_writelane_b32 v31, s7, 13
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 14
-; SI-NEXT: v_writelane_b32 v31, s7, 15
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 16
-; SI-NEXT: v_writelane_b32 v31, s7, 17
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 18
-; SI-NEXT: v_writelane_b32 v31, s7, 19
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 20
-; SI-NEXT: v_writelane_b32 v31, s7, 21
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 22
-; SI-NEXT: v_writelane_b32 v31, s7, 23
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 24
-; SI-NEXT: v_writelane_b32 v31, s7, 25
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: v_writelane_b32 v31, s6, 26
-; SI-NEXT: v_writelane_b32 v31, s7, 27
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB57_2
;
; VI-LABEL: bitcast_v60i16_to_v60f16_scalar:
; VI: ; %bb.0:
@@ -47774,10 +48549,8 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: v_writelane_b32 v30, s35, 1
; VI-NEXT: v_writelane_b32 v30, s36, 2
; VI-NEXT: v_writelane_b32 v30, s37, 3
-; VI-NEXT: v_writelane_b32 v30, s38, 4
-; VI-NEXT: v_writelane_b32 v30, s39, 5
-; VI-NEXT: v_writelane_b32 v30, s30, 6
-; VI-NEXT: v_writelane_b32 v30, s31, 7
+; VI-NEXT: v_writelane_b32 v30, s30, 4
+; VI-NEXT: v_writelane_b32 v30, s31, 5
; VI-NEXT: v_readfirstlane_b32 s7, v15
; VI-NEXT: v_readfirstlane_b32 s8, v14
; VI-NEXT: v_readfirstlane_b32 s10, v13
@@ -47792,8 +48565,8 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s74, v4
; VI-NEXT: v_readfirstlane_b32 s77, v3
; VI-NEXT: v_readfirstlane_b32 s89, v2
-; VI-NEXT: v_readfirstlane_b32 s30, v1
-; VI-NEXT: v_readfirstlane_b32 s35, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v1
+; VI-NEXT: v_readfirstlane_b32 s31, v0
; VI-NEXT: s_lshr_b32 s44, s29, 16
; VI-NEXT: s_lshr_b32 s47, s28, 16
; VI-NEXT: s_lshr_b32 s58, s27, 16
@@ -47804,10 +48577,10 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s79, s22, 16
; VI-NEXT: s_lshr_b32 s88, s21, 16
; VI-NEXT: s_lshr_b32 s91, s20, 16
-; VI-NEXT: s_lshr_b32 s34, s19, 16
-; VI-NEXT: s_lshr_b32 s37, s18, 16
-; VI-NEXT: s_lshr_b32 s38, s17, 16
-; VI-NEXT: s_lshr_b32 s39, s16, 16
+; VI-NEXT: s_lshr_b32 s30, s19, 16
+; VI-NEXT: s_lshr_b32 s35, s18, 16
+; VI-NEXT: s_lshr_b32 s36, s17, 16
+; VI-NEXT: s_lshr_b32 s37, s16, 16
; VI-NEXT: s_lshr_b32 s6, s7, 16
; VI-NEXT: s_lshr_b32 s9, s8, 16
; VI-NEXT: s_lshr_b32 s11, s10, 16
@@ -47822,22 +48595,30 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_lshr_b32 s75, s74, 16
; VI-NEXT: s_lshr_b32 s78, s77, 16
; VI-NEXT: s_lshr_b32 s90, s89, 16
-; VI-NEXT: s_lshr_b32 s31, s30, 16
-; VI-NEXT: s_lshr_b32 s36, s35, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
+; VI-NEXT: s_lshr_b32 s34, s31, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_4
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_3
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: s_add_i32 s39, s39, 3
+; VI-NEXT: s_add_i32 s37, s37, 3
; VI-NEXT: s_add_i32 s17, s17, 3
-; VI-NEXT: s_add_i32 s38, s38, 3
+; VI-NEXT: s_add_i32 s36, s36, 3
; VI-NEXT: s_add_i32 s18, s18, 3
-; VI-NEXT: s_add_i32 s37, s37, 3
+; VI-NEXT: s_add_i32 s35, s35, 3
; VI-NEXT: s_add_i32 s19, s19, 3
-; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s20, s20, 3
; VI-NEXT: s_add_i32 s91, s91, 3
; VI-NEXT: s_add_i32 s21, s21, 3
@@ -47858,10 +48639,10 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s47, s47, 3
; VI-NEXT: s_add_i32 s29, s29, 3
; VI-NEXT: s_add_i32 s44, s44, 3
-; VI-NEXT: s_add_i32 s35, s35, 3
-; VI-NEXT: s_add_i32 s36, s36, 3
-; VI-NEXT: s_add_i32 s30, s30, 3
; VI-NEXT: s_add_i32 s31, s31, 3
+; VI-NEXT: s_add_i32 s34, s34, 3
+; VI-NEXT: s_add_i32 vcc_lo, vcc_lo, 3
+; VI-NEXT: s_add_i32 vcc_hi, vcc_hi, 3
; VI-NEXT: s_add_i32 s89, s89, 3
; VI-NEXT: s_add_i32 s90, s90, 3
; VI-NEXT: s_add_i32 s77, s77, 3
@@ -47890,18 +48671,18 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_add_i32 s9, s9, 3
; VI-NEXT: s_add_i32 s7, s7, 3
; VI-NEXT: s_add_i32 s6, s6, 3
-; VI-NEXT: .LBB57_3: ; %end
+; VI-NEXT: .LBB57_5: ; %end
; VI-NEXT: s_and_b32 s4, 0xffff, s16
-; VI-NEXT: s_lshl_b32 s5, s39, 16
+; VI-NEXT: s_lshl_b32 s5, s37, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_and_b32 s5, 0xffff, s17
-; VI-NEXT: s_lshl_b32 s16, s38, 16
+; VI-NEXT: s_lshl_b32 s16, s36, 16
; VI-NEXT: s_or_b32 s5, s5, s16
; VI-NEXT: s_and_b32 s16, 0xffff, s18
-; VI-NEXT: s_lshl_b32 s17, s37, 16
+; VI-NEXT: s_lshl_b32 s17, s35, 16
; VI-NEXT: s_or_b32 s16, s16, s17
; VI-NEXT: s_and_b32 s17, 0xffff, s19
-; VI-NEXT: s_lshl_b32 s18, s34, 16
+; VI-NEXT: s_lshl_b32 s18, s30, 16
; VI-NEXT: s_or_b32 s17, s17, s18
; VI-NEXT: s_and_b32 s18, 0xffff, s20
; VI-NEXT: s_lshl_b32 s19, s91, 16
@@ -47933,11 +48714,11 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_and_b32 s27, 0xffff, s29
; VI-NEXT: s_lshl_b32 s28, s44, 16
; VI-NEXT: s_or_b32 s27, s27, s28
-; VI-NEXT: s_and_b32 s28, 0xffff, s35
-; VI-NEXT: s_lshl_b32 s29, s36, 16
+; VI-NEXT: s_and_b32 s28, 0xffff, s31
+; VI-NEXT: s_lshl_b32 s29, s34, 16
; VI-NEXT: s_or_b32 s28, s28, s29
-; VI-NEXT: s_and_b32 s29, 0xffff, s30
-; VI-NEXT: s_lshl_b32 s44, s31, 16
+; VI-NEXT: s_and_b32 s29, 0xffff, vcc_lo
+; VI-NEXT: s_lshl_b32 s44, vcc_hi, 16
; VI-NEXT: s_or_b32 s29, s29, s44
; VI-NEXT: s_and_b32 s44, 0xffff, s89
; VI-NEXT: s_lshl_b32 s47, s90, 16
@@ -47981,7 +48762,7 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_or_b32 s10, s10, s11
; VI-NEXT: s_or_b32 s8, s8, s9
; VI-NEXT: s_or_b32 s6, s7, s6
-; VI-NEXT: v_readlane_b32 s30, v30, 6
+; VI-NEXT: v_readlane_b32 s30, v30, 4
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s16
@@ -48012,9 +48793,7 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v27, s10
; VI-NEXT: v_mov_b32_e32 v28, s8
; VI-NEXT: v_mov_b32_e32 v29, s6
-; VI-NEXT: v_readlane_b32 s31, v30, 7
-; VI-NEXT: v_readlane_b32 s39, v30, 5
-; VI-NEXT: v_readlane_b32 s38, v30, 4
+; VI-NEXT: v_readlane_b32 s31, v30, 5
; VI-NEXT: v_readlane_b32 s37, v30, 3
; VI-NEXT: v_readlane_b32 s36, v30, 2
; VI-NEXT: v_readlane_b32 s35, v30, 1
@@ -48024,8 +48803,6 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; VI-NEXT: s_mov_b64 exec, s[4:5]
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_4:
-; VI-NEXT: s_branch .LBB57_2
;
; GFX9-LABEL: bitcast_v60i16_to_v60f16_scalar:
; GFX9: ; %bb.0:
@@ -48045,14 +48822,12 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
-; GFX9-NEXT: v_readfirstlane_b32 s35, v15
-; GFX9-NEXT: v_readfirstlane_b32 s34, v14
-; GFX9-NEXT: v_readfirstlane_b32 s31, v13
-; GFX9-NEXT: v_readfirstlane_b32 s30, v12
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
+; GFX9-NEXT: v_readfirstlane_b32 s31, v15
+; GFX9-NEXT: v_readfirstlane_b32 s30, v14
+; GFX9-NEXT: v_readfirstlane_b32 vcc_hi, v13
+; GFX9-NEXT: v_readfirstlane_b32 vcc_lo, v12
; GFX9-NEXT: v_readfirstlane_b32 s95, v11
; GFX9-NEXT: v_readfirstlane_b32 s94, v10
; GFX9-NEXT: v_readfirstlane_b32 s93, v9
@@ -48079,10 +48854,10 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s8, s18, 16
; GFX9-NEXT: s_lshr_b32 s7, s17, 16
; GFX9-NEXT: s_lshr_b32 s6, s16, 16
-; GFX9-NEXT: s_lshr_b32 s75, s35, 16
-; GFX9-NEXT: s_lshr_b32 s74, s34, 16
-; GFX9-NEXT: s_lshr_b32 s73, s31, 16
-; GFX9-NEXT: s_lshr_b32 s72, s30, 16
+; GFX9-NEXT: s_lshr_b32 s75, s31, 16
+; GFX9-NEXT: s_lshr_b32 s74, s30, 16
+; GFX9-NEXT: s_lshr_b32 s73, vcc_hi, 16
+; GFX9-NEXT: s_lshr_b32 s72, vcc_lo, 16
; GFX9-NEXT: s_lshr_b32 s63, s95, 16
; GFX9-NEXT: s_lshr_b32 s62, s94, 16
; GFX9-NEXT: s_lshr_b32 s61, s93, 16
@@ -48097,17 +48872,25 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s76, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s35, s75
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s31, s75
; GFX9-NEXT: v_pk_add_u16 v29, s4, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s34, s74
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s30, s74
; GFX9-NEXT: v_pk_add_u16 v28, s4, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s31, s73
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, vcc_hi, s73
; GFX9-NEXT: v_pk_add_u16 v27, s4, 3 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s30, s72
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, vcc_lo, s72
; GFX9-NEXT: v_pk_add_u16 v26, s4, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s95, s63
; GFX9-NEXT: v_pk_add_u16 v25, s4, 3 op_sel_hi:[1,0]
@@ -48191,14 +48974,12 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v27
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; GFX9-NEXT: s_branch .LBB57_5
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
-; GFX9-NEXT: v_mov_b32_e32 v29, s35
-; GFX9-NEXT: v_mov_b32_e32 v28, s34
-; GFX9-NEXT: v_mov_b32_e32 v27, s31
-; GFX9-NEXT: v_mov_b32_e32 v26, s30
+; GFX9-NEXT: s_branch .LBB57_6
+; GFX9-NEXT: .LBB57_5:
+; GFX9-NEXT: v_mov_b32_e32 v29, s31
+; GFX9-NEXT: v_mov_b32_e32 v28, s30
+; GFX9-NEXT: v_mov_b32_e32 v27, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v26, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v25, s95
; GFX9-NEXT: v_mov_b32_e32 v24, s94
; GFX9-NEXT: v_mov_b32_e32 v23, s93
@@ -48255,7 +49036,7 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v57, s8
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
-; GFX9-NEXT: .LBB57_5: ; %end
+; GFX9-NEXT: .LBB57_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -48310,7 +49091,7 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
@@ -48329,9 +49110,7 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -48386,11 +49165,16 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s74, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s94, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s94, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-TRUE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s94, -1
+; GFX11-TRUE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s89, s59
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s62, s92, s62
; GFX11-TRUE16-NEXT: v_pk_add_u16 v27, s59, 3 op_sel_hi:[1,0]
@@ -48482,8 +49266,6 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v28
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v29
; GFX11-TRUE16-NEXT: s_branch .LBB57_5
-; GFX11-TRUE16-NEXT: .LBB57_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB57_2
; GFX11-TRUE16-NEXT: .LBB57_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s93 :: v_dual_mov_b32 v28, s92
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s89 :: v_dual_mov_b32 v26, s91
@@ -48598,11 +49380,16 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s74, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s94, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s94, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-FAKE16-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s94, -1
+; GFX11-FAKE16-NEXT: .LBB57_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s59, s89, s59
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s62, s92, s62
; GFX11-FAKE16-NEXT: v_pk_add_u16 v27, s59, 3 op_sel_hi:[1,0]
@@ -48694,8 +49481,6 @@ define inreg <60 x half> @bitcast_v60i16_to_v60f16_scalar(<60 x i16> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 16, v26
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX11-FAKE16-NEXT: s_branch .LBB57_5
-; GFX11-FAKE16-NEXT: .LBB57_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB57_2
; GFX11-FAKE16-NEXT: .LBB57_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s93 :: v_dual_mov_b32 v26, s92
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s89 :: v_dual_mov_b32 v28, s91
@@ -49855,10 +50640,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
-; SI-NEXT: v_writelane_b32 v63, s34, 0
-; SI-NEXT: v_writelane_b32 v63, s35, 1
-; SI-NEXT: v_writelane_b32 v63, s30, 2
-; SI-NEXT: v_writelane_b32 v63, s31, 3
+; SI-NEXT: v_writelane_b32 v63, s30, 0
+; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: v_readfirstlane_b32 s7, v14
; SI-NEXT: v_readfirstlane_b32 s47, v13
@@ -49874,7 +50657,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_readfirstlane_b32 s41, v3
; SI-NEXT: v_readfirstlane_b32 s94, v2
; SI-NEXT: v_readfirstlane_b32 s40, v1
-; SI-NEXT: v_readfirstlane_b32 s30, v0
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v0
; SI-NEXT: s_lshr_b32 s15, s29, 16
; SI-NEXT: s_lshr_b32 s79, s28, 16
; SI-NEXT: s_lshr_b32 s14, s27, 16
@@ -49884,11 +50667,11 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s12, s23, 16
; SI-NEXT: s_lshr_b32 s95, s22, 16
; SI-NEXT: s_lshr_b32 s11, s21, 16
-; SI-NEXT: s_lshr_b32 s31, s20, 16
+; SI-NEXT: s_lshr_b32 vcc_hi, s20, 16
; SI-NEXT: s_lshr_b32 s10, s19, 16
-; SI-NEXT: s_lshr_b32 s34, s18, 16
+; SI-NEXT: s_lshr_b32 s30, s18, 16
; SI-NEXT: s_lshr_b32 s9, s17, 16
-; SI-NEXT: s_lshr_b32 s35, s16, 16
+; SI-NEXT: s_lshr_b32 s31, s16, 16
; SI-NEXT: s_lshr_b32 s75, s6, 16
; SI-NEXT: s_lshr_b32 s57, s7, 16
; SI-NEXT: s_lshr_b32 s73, s47, 16
@@ -49904,16 +50687,24 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s56, s41, 16
; SI-NEXT: s_lshr_b32 s89, s94, 16
; SI-NEXT: s_lshr_b32 s46, s40, 16
-; SI-NEXT: s_lshr_b32 s92, s30, 16
+; SI-NEXT: s_lshr_b32 s92, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_3
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB59_4
-; SI-NEXT: .LBB59_2: ; %cmp.true
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s35
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
-; SI-NEXT: v_cvt_f32_f16_e32 v2, s34
+; SI-NEXT: v_cvt_f32_f16_e32 v2, s30
; SI-NEXT: v_cvt_f32_f16_e32 v3, s18
; SI-NEXT: v_add_f32_e32 v0, 0x38000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -49921,7 +50712,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v2
; SI-NEXT: v_lshlrev_b32_e32 v46, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s31
+; SI-NEXT: v_cvt_f32_f16_e32 v0, vcc_hi
; SI-NEXT: v_or_b32_e32 v25, v1, v46
; SI-NEXT: v_cvt_f16_f32_e32 v1, v2
; SI-NEXT: v_add_f32_e32 v2, 0x38000000, v3
@@ -49969,7 +50760,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v1, s92
; SI-NEXT: v_lshlrev_b32_e32 v54, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s30
+; SI-NEXT: v_cvt_f32_f16_e32 v0, vcc_lo
; SI-NEXT: v_or_b32_e32 v21, v3, v54
; SI-NEXT: v_cvt_f32_f16_e32 v3, s89
; SI-NEXT: v_add_f32_e32 v1, 0x38000000, v1
@@ -50210,10 +51001,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v23, v14
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_branch .LBB59_5
-; SI-NEXT: .LBB59_3:
-; SI-NEXT: s_branch .LBB59_2
-; SI-NEXT: .LBB59_4:
+; SI-NEXT: s_branch .LBB59_6
+; SI-NEXT: .LBB59_5:
; SI-NEXT: v_mov_b32_e32 v11, s13
; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: v_mov_b32_e32 v12, s45
@@ -50239,7 +51028,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v43, s88
; SI-NEXT: v_mov_b32_e32 v21, s91
; SI-NEXT: v_mov_b32_e32 v33, s94
-; SI-NEXT: v_mov_b32_e32 v55, s30
+; SI-NEXT: v_mov_b32_e32 v55, vcc_lo
; SI-NEXT: v_mov_b32_e32 v59, s28
; SI-NEXT: v_mov_b32_e32 v57, s26
; SI-NEXT: v_mov_b32_e32 v10, s24
@@ -50263,9 +51052,9 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v44, s47
; SI-NEXT: v_mov_b32_e32 v29, s6
-; SI-NEXT: v_mov_b32_e32 v58, s35
-; SI-NEXT: v_mov_b32_e32 v56, s34
-; SI-NEXT: v_mov_b32_e32 v46, s31
+; SI-NEXT: v_mov_b32_e32 v58, s31
+; SI-NEXT: v_mov_b32_e32 v56, s30
+; SI-NEXT: v_mov_b32_e32 v46, vcc_hi
; SI-NEXT: v_mov_b32_e32 v24, s95
; SI-NEXT: v_mov_b32_e32 v32, s93
; SI-NEXT: v_mov_b32_e32 v40, s90
@@ -50279,7 +51068,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v60, s59
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: .LBB59_5: ; %end
+; SI-NEXT: .LBB59_6: ; %end
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v58
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -50368,10 +51157,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; SI-NEXT: v_readlane_b32 s30, v63, 2
-; SI-NEXT: v_readlane_b32 s31, v63, 3
-; SI-NEXT: v_readlane_b32 s35, v63, 1
-; SI-NEXT: v_readlane_b32 s34, v63, 0
+; SI-NEXT: v_readlane_b32 s30, v63, 0
+; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v26
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v28
@@ -50430,10 +51217,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_writelane_b32 v60, s35, 1
; VI-NEXT: v_writelane_b32 v60, s36, 2
; VI-NEXT: v_writelane_b32 v60, s37, 3
-; VI-NEXT: v_writelane_b32 v60, s38, 4
-; VI-NEXT: v_writelane_b32 v60, s39, 5
-; VI-NEXT: v_writelane_b32 v60, s30, 6
-; VI-NEXT: v_writelane_b32 v60, s31, 7
+; VI-NEXT: v_writelane_b32 v60, s30, 4
+; VI-NEXT: v_writelane_b32 v60, s31, 5
; VI-NEXT: v_readfirstlane_b32 s44, v15
; VI-NEXT: v_readfirstlane_b32 s46, v14
; VI-NEXT: v_readfirstlane_b32 s56, v13
@@ -50446,10 +51231,10 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s78, v6
; VI-NEXT: v_readfirstlane_b32 s88, v5
; VI-NEXT: v_readfirstlane_b32 s90, v4
-; VI-NEXT: v_readfirstlane_b32 s30, v3
-; VI-NEXT: v_readfirstlane_b32 s34, v2
-; VI-NEXT: v_readfirstlane_b32 s36, v1
-; VI-NEXT: v_readfirstlane_b32 s38, v0
+; VI-NEXT: v_readfirstlane_b32 vcc_lo, v3
+; VI-NEXT: v_readfirstlane_b32 s30, v2
+; VI-NEXT: v_readfirstlane_b32 s34, v1
+; VI-NEXT: v_readfirstlane_b32 s36, v0
; VI-NEXT: s_lshr_b32 s6, s29, 16
; VI-NEXT: s_lshr_b32 s7, s28, 16
; VI-NEXT: s_lshr_b32 s8, s27, 16
@@ -50476,16 +51261,24 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s79, s78, 16
; VI-NEXT: s_lshr_b32 s89, s88, 16
; VI-NEXT: s_lshr_b32 s91, s90, 16
+; VI-NEXT: s_lshr_b32 vcc_hi, vcc_lo, 16
; VI-NEXT: s_lshr_b32 s31, s30, 16
; VI-NEXT: s_lshr_b32 s35, s34, 16
; VI-NEXT: s_lshr_b32 s37, s36, 16
-; VI-NEXT: s_lshr_b32 s39, s38, 16
; VI-NEXT: v_readfirstlane_b32 s4, v16
; VI-NEXT: s_cmp_lg_u32 s4, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_3
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_4
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_mov_b32_e32 v30, 0x200
; VI-NEXT: v_add_f16_e32 v0, s16, v30
; VI-NEXT: v_add_f16_e32 v59, s43, v30
@@ -50515,14 +51308,14 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v55, s7, v30
; VI-NEXT: v_add_f16_e32 v13, s29, v30
; VI-NEXT: v_add_f16_e32 v54, s6, v30
-; VI-NEXT: v_add_f16_e32 v14, s38, v30
-; VI-NEXT: v_add_f16_e32 v53, s39, v30
-; VI-NEXT: v_add_f16_e32 v15, s36, v30
-; VI-NEXT: v_add_f16_e32 v52, s37, v30
-; VI-NEXT: v_add_f16_e32 v16, s34, v30
-; VI-NEXT: v_add_f16_e32 v51, s35, v30
-; VI-NEXT: v_add_f16_e32 v17, s30, v30
-; VI-NEXT: v_add_f16_e32 v50, s31, v30
+; VI-NEXT: v_add_f16_e32 v14, s36, v30
+; VI-NEXT: v_add_f16_e32 v53, s37, v30
+; VI-NEXT: v_add_f16_e32 v15, s34, v30
+; VI-NEXT: v_add_f16_e32 v52, s35, v30
+; VI-NEXT: v_add_f16_e32 v16, s30, v30
+; VI-NEXT: v_add_f16_e32 v51, s31, v30
+; VI-NEXT: v_add_f16_e32 v17, vcc_lo, v30
+; VI-NEXT: v_add_f16_e32 v50, vcc_hi, v30
; VI-NEXT: v_add_f16_e32 v18, s90, v30
; VI-NEXT: v_add_f16_e32 v49, s91, v30
; VI-NEXT: v_add_f16_e32 v19, s88, v30
@@ -50547,10 +51340,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_add_f16_e32 v31, s47, v30
; VI-NEXT: v_add_f16_e32 v29, s44, v30
; VI-NEXT: v_add_f16_e32 v30, s45, v30
-; VI-NEXT: s_branch .LBB59_5
-; VI-NEXT: .LBB59_3:
-; VI-NEXT: s_branch .LBB59_2
-; VI-NEXT: .LBB59_4:
+; VI-NEXT: s_branch .LBB59_6
+; VI-NEXT: .LBB59_5:
; VI-NEXT: v_mov_b32_e32 v30, s45
; VI-NEXT: v_mov_b32_e32 v29, s44
; VI-NEXT: v_mov_b32_e32 v31, s47
@@ -50575,14 +51366,14 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v19, s88
; VI-NEXT: v_mov_b32_e32 v49, s91
; VI-NEXT: v_mov_b32_e32 v18, s90
-; VI-NEXT: v_mov_b32_e32 v50, s31
-; VI-NEXT: v_mov_b32_e32 v17, s30
-; VI-NEXT: v_mov_b32_e32 v51, s35
-; VI-NEXT: v_mov_b32_e32 v16, s34
-; VI-NEXT: v_mov_b32_e32 v52, s37
-; VI-NEXT: v_mov_b32_e32 v15, s36
-; VI-NEXT: v_mov_b32_e32 v53, s39
-; VI-NEXT: v_mov_b32_e32 v14, s38
+; VI-NEXT: v_mov_b32_e32 v50, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v17, vcc_lo
+; VI-NEXT: v_mov_b32_e32 v51, s31
+; VI-NEXT: v_mov_b32_e32 v16, s30
+; VI-NEXT: v_mov_b32_e32 v52, s35
+; VI-NEXT: v_mov_b32_e32 v15, s34
+; VI-NEXT: v_mov_b32_e32 v53, s37
+; VI-NEXT: v_mov_b32_e32 v14, s36
; VI-NEXT: v_mov_b32_e32 v54, s6
; VI-NEXT: v_mov_b32_e32 v13, s29
; VI-NEXT: v_mov_b32_e32 v55, s7
@@ -50611,7 +51402,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v59, s43
; VI-NEXT: v_mov_b32_e32 v0, s16
-; VI-NEXT: .LBB59_5: ; %end
+; VI-NEXT: .LBB59_6: ; %end
; VI-NEXT: v_lshlrev_b32_e32 v59, 16, v59
; VI-NEXT: v_lshlrev_b32_e32 v58, 16, v58
; VI-NEXT: v_lshlrev_b32_e32 v57, 16, v57
@@ -50666,7 +51457,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; VI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
; VI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; VI-NEXT: v_readlane_b32 s30, v60, 6
+; VI-NEXT: v_readlane_b32 s30, v60, 4
; VI-NEXT: v_or_b32_sdwa v12, v12, v55 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v13, v13, v54 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v14, v14, v53 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -50685,9 +51476,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; VI-NEXT: v_or_b32_sdwa v27, v27, v32 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v28, v28, v31 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_or_b32_sdwa v29, v29, v30 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: v_readlane_b32 s31, v60, 7
-; VI-NEXT: v_readlane_b32 s39, v60, 5
-; VI-NEXT: v_readlane_b32 s38, v60, 4
+; VI-NEXT: v_readlane_b32 s31, v60, 5
; VI-NEXT: v_readlane_b32 s37, v60, 3
; VI-NEXT: v_readlane_b32 s36, v60, 2
; VI-NEXT: v_readlane_b32 s35, v60, 1
@@ -50716,14 +51505,12 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-NEXT: v_writelane_b32 v60, s34, 0
-; GFX9-NEXT: v_writelane_b32 v60, s35, 1
-; GFX9-NEXT: v_writelane_b32 v60, s30, 2
-; GFX9-NEXT: v_writelane_b32 v60, s31, 3
-; GFX9-NEXT: v_readfirstlane_b32 s35, v15
-; GFX9-NEXT: v_readfirstlane_b32 s34, v14
-; GFX9-NEXT: v_readfirstlane_b32 s31, v13
-; GFX9-NEXT: v_readfirstlane_b32 s30, v12
+; GFX9-NEXT: v_writelane_b32 v60, s30, 0
+; GFX9-NEXT: v_writelane_b32 v60, s31, 1
+; GFX9-NEXT: v_readfirstlane_b32 s31, v15
+; GFX9-NEXT: v_readfirstlane_b32 s30, v14
+; GFX9-NEXT: v_readfirstlane_b32 vcc_hi, v13
+; GFX9-NEXT: v_readfirstlane_b32 vcc_lo, v12
; GFX9-NEXT: v_readfirstlane_b32 s95, v11
; GFX9-NEXT: v_readfirstlane_b32 s94, v10
; GFX9-NEXT: v_readfirstlane_b32 s93, v9
@@ -50750,10 +51537,10 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s8, s18, 16
; GFX9-NEXT: s_lshr_b32 s7, s17, 16
; GFX9-NEXT: s_lshr_b32 s6, s16, 16
-; GFX9-NEXT: s_lshr_b32 s75, s35, 16
-; GFX9-NEXT: s_lshr_b32 s74, s34, 16
-; GFX9-NEXT: s_lshr_b32 s73, s31, 16
-; GFX9-NEXT: s_lshr_b32 s72, s30, 16
+; GFX9-NEXT: s_lshr_b32 s75, s31, 16
+; GFX9-NEXT: s_lshr_b32 s74, s30, 16
+; GFX9-NEXT: s_lshr_b32 s73, vcc_hi, 16
+; GFX9-NEXT: s_lshr_b32 s72, vcc_lo, 16
; GFX9-NEXT: s_lshr_b32 s63, s95, 16
; GFX9-NEXT: s_lshr_b32 s62, s94, 16
; GFX9-NEXT: s_lshr_b32 s61, s93, 16
@@ -50768,18 +51555,26 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: s_lshr_b32 s44, s76, 16
; GFX9-NEXT: v_readfirstlane_b32 s4, v16
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s35, s75
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s31, s75
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v29, s4, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s34, s74
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, s30, s74
; GFX9-NEXT: v_pk_add_f16 v28, s4, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s31, s73
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, vcc_hi, s73
; GFX9-NEXT: v_pk_add_f16 v27, s4, v0 op_sel_hi:[1,0]
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s30, s72
+; GFX9-NEXT: s_pack_ll_b32_b16 s4, vcc_lo, s72
; GFX9-NEXT: v_pk_add_f16 v26, s4, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_pack_ll_b32_b16 s4, s95, s63
; GFX9-NEXT: v_pk_add_f16 v25, s4, v0 op_sel_hi:[1,0]
@@ -50863,14 +51658,12 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v27
; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v29
-; GFX9-NEXT: s_branch .LBB59_5
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
-; GFX9-NEXT: v_mov_b32_e32 v29, s35
-; GFX9-NEXT: v_mov_b32_e32 v28, s34
-; GFX9-NEXT: v_mov_b32_e32 v27, s31
-; GFX9-NEXT: v_mov_b32_e32 v26, s30
+; GFX9-NEXT: s_branch .LBB59_6
+; GFX9-NEXT: .LBB59_5:
+; GFX9-NEXT: v_mov_b32_e32 v29, s31
+; GFX9-NEXT: v_mov_b32_e32 v28, s30
+; GFX9-NEXT: v_mov_b32_e32 v27, vcc_hi
+; GFX9-NEXT: v_mov_b32_e32 v26, vcc_lo
; GFX9-NEXT: v_mov_b32_e32 v25, s95
; GFX9-NEXT: v_mov_b32_e32 v24, s94
; GFX9-NEXT: v_mov_b32_e32 v23, s93
@@ -50927,7 +51720,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v57, s8
; GFX9-NEXT: v_mov_b32_e32 v58, s7
; GFX9-NEXT: v_mov_b32_e32 v59, s6
-; GFX9-NEXT: .LBB59_5: ; %end
+; GFX9-NEXT: .LBB59_6: ; %end
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -50982,7 +51775,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff, v27
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff, v28
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff, v29
-; GFX9-NEXT: v_readlane_b32 s30, v60, 2
+; GFX9-NEXT: v_readlane_b32 s30, v60, 0
; GFX9-NEXT: v_lshl_or_b32 v12, v55, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v13, v54, 16, v13
; GFX9-NEXT: v_lshl_or_b32 v14, v53, 16, v14
@@ -51001,9 +51794,7 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX9-NEXT: v_lshl_or_b32 v27, v32, 16, v27
; GFX9-NEXT: v_lshl_or_b32 v28, v31, 16, v28
; GFX9-NEXT: v_lshl_or_b32 v29, v30, 16, v29
-; GFX9-NEXT: v_readlane_b32 s31, v60, 3
-; GFX9-NEXT: v_readlane_b32 s35, v60, 1
-; GFX9-NEXT: v_readlane_b32 s34, v60, 0
+; GFX9-NEXT: v_readlane_b32 s31, v60, 1
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
@@ -51058,11 +51849,16 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX11-TRUE16-NEXT: s_lshr_b32 s46, s74, 16
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s94, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s94, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-TRUE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s94, -1
+; GFX11-TRUE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s59, s89, s59
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s62, s92, s62
; GFX11-TRUE16-NEXT: v_pk_add_f16 v27, 0x200, s59 op_sel_hi:[0,1]
@@ -51154,8 +51950,6 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v28
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v29
; GFX11-TRUE16-NEXT: s_branch .LBB59_5
-; GFX11-TRUE16-NEXT: .LBB59_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB59_2
; GFX11-TRUE16-NEXT: .LBB59_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s93 :: v_dual_mov_b32 v28, s92
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s89 :: v_dual_mov_b32 v26, s91
@@ -51270,11 +52064,16 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX11-FAKE16-NEXT: s_lshr_b32 s46, s74, 16
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s94, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s94, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s94
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-FAKE16-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s94, -1
+; GFX11-FAKE16-NEXT: .LBB59_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s94, s94, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s94, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s94, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s59, s89, s59
; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s62, s92, s62
; GFX11-FAKE16-NEXT: v_pk_add_f16 v27, 0x200, s59 op_sel_hi:[0,1]
@@ -51366,8 +52165,6 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v31, 16, v26
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v30, 16, v25
; GFX11-FAKE16-NEXT: s_branch .LBB59_5
-; GFX11-FAKE16-NEXT: .LBB59_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB59_2
; GFX11-FAKE16-NEXT: .LBB59_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v25, s93 :: v_dual_mov_b32 v26, s92
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v27, s89 :: v_dual_mov_b32 v28, s91
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index a1c0db086bf3c..16f9dc4083053 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -89,79 +89,100 @@ define inreg <3 x float> @bitcast_v3i32_to_v3f32_scalar(<3 x i32> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB1_4
+; SI-NEXT: s_cbranch_scc0 .LBB1_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB1_3
-; SI-NEXT: .LBB1_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB1_3
+; SI-NEXT: .LBB1_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB1_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
-; SI-NEXT: .LBB1_3: ; %end
+; SI-NEXT: .LBB1_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB1_4:
-; SI-NEXT: s_branch .LBB1_2
;
; VI-LABEL: bitcast_v3i32_to_v3f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB1_4
+; VI-NEXT: s_cbranch_scc0 .LBB1_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB1_3
-; VI-NEXT: .LBB1_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB1_3
+; VI-NEXT: .LBB1_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB1_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB1_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB1_3: ; %end
+; VI-NEXT: .LBB1_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB1_4:
-; VI-NEXT: s_branch .LBB1_2
;
; GFX9-LABEL: bitcast_v3i32_to_v3f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB1_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB1_3
-; GFX9-NEXT: .LBB1_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB1_3
+; GFX9-NEXT: .LBB1_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB1_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB1_3: ; %end
+; GFX9-NEXT: .LBB1_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB1_4:
-; GFX9-NEXT: s_branch .LBB1_2
;
; GFX11-LABEL: bitcast_v3i32_to_v3f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB1_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB1_3
-; GFX11-NEXT: .LBB1_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB1_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB1_3: ; %end
+; GFX11-NEXT: .LBB1_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB1_4:
-; GFX11-NEXT: s_branch .LBB1_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -261,17 +282,23 @@ define inreg <3 x i32> @bitcast_v3f32_to_v3i32_scalar(<3 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB3_3
+; SI-NEXT: s_cbranch_scc0 .LBB3_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB3_4
-; SI-NEXT: .LBB3_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB3_3
+; SI-NEXT: .LBB3_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB3_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB3_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB3_3:
-; SI-NEXT: s_branch .LBB3_2
-; SI-NEXT: .LBB3_4:
+; SI-NEXT: .LBB3_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
@@ -281,17 +308,23 @@ define inreg <3 x i32> @bitcast_v3f32_to_v3i32_scalar(<3 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB3_3
+; VI-NEXT: s_cbranch_scc0 .LBB3_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB3_4
-; VI-NEXT: .LBB3_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB3_3
+; VI-NEXT: .LBB3_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB3_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB3_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB3_3:
-; VI-NEXT: s_branch .LBB3_2
-; VI-NEXT: .LBB3_4:
+; VI-NEXT: .LBB3_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -301,17 +334,23 @@ define inreg <3 x i32> @bitcast_v3f32_to_v3i32_scalar(<3 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB3_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB3_4
-; GFX9-NEXT: .LBB3_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB3_3
+; GFX9-NEXT: .LBB3_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB3_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB3_3:
-; GFX9-NEXT: s_branch .LBB3_2
-; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: .LBB3_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -322,17 +361,20 @@ define inreg <3 x i32> @bitcast_v3f32_to_v3i32_scalar(<3 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB3_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB3_4
-; GFX11-NEXT: .LBB3_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB3_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB3_3:
-; GFX11-NEXT: s_branch .LBB3_2
; GFX11-NEXT: .LBB3_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -630,7 +672,7 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB5_4
+; SI-NEXT: s_cbranch_scc0 .LBB5_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s19, s17, 24
; SI-NEXT: s_lshr_b32 s22, s17, 16
@@ -641,8 +683,25 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB5_3
-; SI-NEXT: .LBB5_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB5_3
+; SI-NEXT: .LBB5_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB5_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB5_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s16, s16, 3
@@ -655,7 +714,7 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; SI-NEXT: s_lshr_b64 s[10:11], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[14:15], s[18:19], 8
-; SI-NEXT: .LBB5_3: ; %end
+; SI-NEXT: .LBB5_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s12
; SI-NEXT: v_mov_b32_e32 v2, s8
@@ -669,23 +728,12 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v10, s10
; SI-NEXT: v_mov_b32_e32 v11, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB5_4:
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB5_2
;
; VI-LABEL: bitcast_v3i32_to_v12i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB5_4
+; VI-NEXT: s_cbranch_scc0 .LBB5_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s19, s16, 8
; VI-NEXT: s_lshr_b32 s10, s18, 16
@@ -696,8 +744,25 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s15, s16, 16
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB5_3
-; VI-NEXT: .LBB5_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB5_3
+; VI-NEXT: .LBB5_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr19
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB5_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB5_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
@@ -710,7 +775,7 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s14, s17, 8
; VI-NEXT: s_lshr_b32 s15, s16, 16
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
-; VI-NEXT: .LBB5_3: ; %end
+; VI-NEXT: .LBB5_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s19
; VI-NEXT: v_mov_b32_e32 v2, s15
@@ -724,23 +789,12 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v10, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB5_4:
-; VI-NEXT: ; implicit-def: $sgpr19
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB5_2
;
; GFX9-LABEL: bitcast_v3i32_to_v12i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s19, s16, 8
; GFX9-NEXT: s_lshr_b32 s10, s18, 16
@@ -751,8 +805,25 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s15, s16, 16
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-NEXT: .LBB5_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB5_3
+; GFX9-NEXT: .LBB5_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr19
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB5_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
@@ -765,7 +836,7 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s14, s17, 8
; GFX9-NEXT: s_lshr_b32 s15, s16, 16
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
-; GFX9-NEXT: .LBB5_3: ; %end
+; GFX9-NEXT: .LBB5_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s19
; GFX9-NEXT: v_mov_b32_e32 v2, s15
@@ -779,24 +850,13 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v10, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB5_4:
-; GFX9-NEXT: ; implicit-def: $sgpr19
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB5_2
;
; GFX11-LABEL: bitcast_v3i32_to_v12i8_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s14, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB5_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB5_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-NEXT: s_lshr_b32 s8, s2, 8
@@ -807,9 +867,25 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s13, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s14
-; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
-; GFX11-NEXT: .LBB5_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB5_3
+; GFX11-NEXT: .LBB5_2:
+; GFX11-NEXT: s_mov_b32 s14, -1
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr8
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB5_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
@@ -822,7 +898,7 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s12, s0, 16
; GFX11-NEXT: s_lshr_b32 s13, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-NEXT: .LBB5_3: ; %end
+; GFX11-NEXT: .LBB5_5: ; %end
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13
; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v3, s4
; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s11
@@ -830,17 +906,6 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s8
; GFX11-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v11, s6
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB5_4:
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr3
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB5_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1289,7 +1354,7 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB7_4
+; SI-NEXT: s_cbranch_scc0 .LBB7_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -1318,8 +1383,17 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_cbranch_execnz .LBB7_3
-; SI-NEXT: .LBB7_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB7_3
+; SI-NEXT: .LBB7_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB7_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB7_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -1359,20 +1433,17 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: .LBB7_3: ; %end
+; SI-NEXT: .LBB7_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: bitcast_v12i8_to_v3i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB7_4
+; VI-NEXT: s_cbranch_scc0 .LBB7_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -1393,8 +1464,17 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v2, s26, v4, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: s_cbranch_execnz .LBB7_3
-; VI-NEXT: .LBB7_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB7_3
+; VI-NEXT: .LBB7_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; VI-NEXT: .LBB7_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB7_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -1426,17 +1506,14 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
-; VI-NEXT: .LBB7_3: ; %end
+; VI-NEXT: .LBB7_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB7_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; VI-NEXT: s_branch .LBB7_2
;
; GFX9-LABEL: bitcast_v12i8_to_v3i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -1457,8 +1534,17 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v2, s26, v4, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: s_cbranch_execnz .LBB7_3
-; GFX9-NEXT: .LBB7_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB7_3
+; GFX9-NEXT: .LBB7_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX9-NEXT: .LBB7_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -1488,18 +1574,15 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB7_3: ; %end
+; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB7_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX9-NEXT: s_branch .LBB7_2
;
; GFX11-LABEL: bitcast_v12i8_to_v3i32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB7_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1517,9 +1600,17 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB7_3
-; GFX11-NEXT: .LBB7_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB7_3
+; GFX11-NEXT: .LBB7_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-NEXT: .LBB7_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -1549,11 +1640,8 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB7_3: ; %end
+; GFX11-NEXT: .LBB7_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB7_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-NEXT: s_branch .LBB7_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1689,7 +1777,7 @@ define inreg <6 x bfloat> @bitcast_v3i32_to_v6bf16_scalar(<3 x i32> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB9_4
+; SI-NEXT: s_cbranch_scc0 .LBB9_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s7, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s18, 16
@@ -1697,8 +1785,22 @@ define inreg <6 x bfloat> @bitcast_v3i32_to_v6bf16_scalar(<3 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s8, s17, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s10, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB9_3
-; SI-NEXT: .LBB9_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB9_3
+; SI-NEXT: .LBB9_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: .LBB9_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB9_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s18, s18, 3
@@ -1708,7 +1810,7 @@ define inreg <6 x bfloat> @bitcast_v3i32_to_v6bf16_scalar(<3 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s8, s17, 16
; SI-NEXT: s_and_b32 s11, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s10, s16, 16
-; SI-NEXT: .LBB9_3: ; %end
+; SI-NEXT: .LBB9_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s11
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s10
@@ -1722,73 +1824,80 @@ define inreg <6 x bfloat> @bitcast_v3i32_to_v6bf16_scalar(<3 x i32> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB9_4:
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: s_branch .LBB9_2
;
; VI-LABEL: bitcast_v3i32_to_v6bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB9_4
+; VI-NEXT: s_cbranch_scc0 .LBB9_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB9_3
-; VI-NEXT: .LBB9_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB9_3
+; VI-NEXT: .LBB9_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB9_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB9_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB9_3: ; %end
+; VI-NEXT: .LBB9_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB9_4:
-; VI-NEXT: s_branch .LBB9_2
;
; GFX9-LABEL: bitcast_v3i32_to_v6bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB9_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB9_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB9_3
-; GFX9-NEXT: .LBB9_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB9_3
+; GFX9-NEXT: .LBB9_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB9_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB9_3: ; %end
+; GFX9-NEXT: .LBB9_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB9_4:
-; GFX9-NEXT: s_branch .LBB9_2
;
; GFX11-LABEL: bitcast_v3i32_to_v6bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB9_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB9_3
-; GFX11-NEXT: .LBB9_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB9_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB9_3: ; %end
+; GFX11-NEXT: .LBB9_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB9_4:
-; GFX11-NEXT: s_branch .LBB9_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2170,7 +2279,7 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB11_4
+; SI-NEXT: s_cbranch_scc0 .LBB11_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
; SI-NEXT: v_lshr_b64 v[0:1], v[8:9], 16
@@ -2178,8 +2287,17 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_cbranch_execnz .LBB11_3
-; SI-NEXT: .LBB11_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB11_3
+; SI-NEXT: .LBB11_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; SI-NEXT: .LBB11_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB11_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -2198,20 +2316,25 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: .LBB11_3: ; %end
+; SI-NEXT: .LBB11_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB11_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; SI-NEXT: s_branch .LBB11_2
;
; VI-LABEL: bitcast_v6bf16_to_v3i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB11_3
+; VI-NEXT: s_cbranch_scc0 .LBB11_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB11_4
-; VI-NEXT: .LBB11_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB11_3
+; VI-NEXT: .LBB11_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB11_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB11_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s18, 16
; VI-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v5
@@ -2269,9 +2392,7 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB11_3:
-; VI-NEXT: s_branch .LBB11_2
-; VI-NEXT: .LBB11_4:
+; VI-NEXT: .LBB11_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2281,10 +2402,18 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB11_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB11_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB11_4
-; GFX9-NEXT: .LBB11_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB11_3
+; GFX9-NEXT: .LBB11_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB11_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -2345,9 +2474,7 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB11_3:
-; GFX9-NEXT: s_branch .LBB11_2
-; GFX9-NEXT: .LBB11_4:
+; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2358,11 +2485,16 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-TRUE16-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: .LBB11_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -2428,8 +2560,6 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB11_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB11_2
; GFX11-TRUE16-NEXT: .LBB11_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s2
@@ -2440,11 +2570,16 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB11_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB11_4
-; GFX11-FAKE16-NEXT: .LBB11_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: .LBB11_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -2512,8 +2647,6 @@ define inreg <3 x i32> @bitcast_v6bf16_to_v3i32_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB11_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB11_2
; GFX11-FAKE16-NEXT: .LBB11_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s2
@@ -2638,20 +2771,31 @@ define inreg <6 x half> @bitcast_v3i32_to_v6f16_scalar(<3 x i32> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB13_4
+; SI-NEXT: s_cbranch_scc0 .LBB13_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB13_3
-; SI-NEXT: .LBB13_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB13_3
+; SI-NEXT: .LBB13_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB13_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB13_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
-; SI-NEXT: .LBB13_3: ; %end
+; SI-NEXT: .LBB13_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -2665,70 +2809,80 @@ define inreg <6 x half> @bitcast_v3i32_to_v6f16_scalar(<3 x i32> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB13_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB13_2
;
; VI-LABEL: bitcast_v3i32_to_v6f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB13_4
+; VI-NEXT: s_cbranch_scc0 .LBB13_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB13_3
-; VI-NEXT: .LBB13_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB13_3
+; VI-NEXT: .LBB13_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB13_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB13_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB13_3: ; %end
+; VI-NEXT: .LBB13_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB13_4:
-; VI-NEXT: s_branch .LBB13_2
;
; GFX9-LABEL: bitcast_v3i32_to_v6f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB13_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB13_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-NEXT: .LBB13_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB13_3
+; GFX9-NEXT: .LBB13_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB13_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB13_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB13_3: ; %end
+; GFX9-NEXT: .LBB13_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB13_4:
-; GFX9-NEXT: s_branch .LBB13_2
;
; GFX11-LABEL: bitcast_v3i32_to_v6f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB13_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB13_3
-; GFX11-NEXT: .LBB13_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB13_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB13_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB13_3: ; %end
+; GFX11-NEXT: .LBB13_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB13_4:
-; GFX11-NEXT: s_branch .LBB13_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2892,7 +3046,7 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_3
+; SI-NEXT: s_cbranch_scc0 .LBB15_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -2903,8 +3057,17 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: s_and_b32 s6, s18, 0xffff
; SI-NEXT: s_lshl_b32 s8, s7, 16
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_cbranch_execnz .LBB15_4
-; SI-NEXT: .LBB15_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB15_3
+; SI-NEXT: .LBB15_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB15_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s11
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s10
@@ -2930,10 +3093,7 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB15_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB15_2
-; SI-NEXT: .LBB15_4:
+; SI-NEXT: .LBB15_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -2943,10 +3103,18 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_3
+; VI-NEXT: s_cbranch_scc0 .LBB15_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB15_4
-; VI-NEXT: .LBB15_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB15_3
+; VI-NEXT: .LBB15_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB15_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB15_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s18, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -2964,9 +3132,7 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB15_3:
-; VI-NEXT: s_branch .LBB15_2
-; VI-NEXT: .LBB15_4:
+; VI-NEXT: .LBB15_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -2976,18 +3142,24 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB15_4
-; GFX9-NEXT: .LBB15_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB15_3
+; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB15_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB15_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB15_3:
-; GFX9-NEXT: s_branch .LBB15_2
-; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -2998,17 +3170,20 @@ define inreg <3 x i32> @bitcast_v6f16_to_v3i32_scalar(<6 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
-; GFX11-NEXT: .LBB15_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB15_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB15_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB15_3:
-; GFX11-NEXT: s_branch .LBB15_2
; GFX11-NEXT: .LBB15_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -3133,20 +3308,31 @@ define inreg <6 x i16> @bitcast_v3i32_to_v6i16_scalar(<3 x i32> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB17_4
+; SI-NEXT: s_cbranch_scc0 .LBB17_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB17_3
-; SI-NEXT: .LBB17_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB17_3
+; SI-NEXT: .LBB17_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB17_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB17_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_add_i32 s17, s17, 3
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
-; SI-NEXT: .LBB17_3: ; %end
+; SI-NEXT: .LBB17_5: ; %end
; SI-NEXT: s_and_b32 s5, s16, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s5, s5, s6
@@ -3160,70 +3346,80 @@ define inreg <6 x i16> @bitcast_v3i32_to_v6i16_scalar(<3 x i32> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB17_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB17_2
;
; VI-LABEL: bitcast_v3i32_to_v6i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB17_4
+; VI-NEXT: s_cbranch_scc0 .LBB17_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB17_3
-; VI-NEXT: .LBB17_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB17_3
+; VI-NEXT: .LBB17_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB17_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB17_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s18, s18, 3
; VI-NEXT: s_add_i32 s17, s17, 3
; VI-NEXT: s_add_i32 s16, s16, 3
-; VI-NEXT: .LBB17_3: ; %end
+; VI-NEXT: .LBB17_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB17_4:
-; VI-NEXT: s_branch .LBB17_2
;
; GFX9-LABEL: bitcast_v3i32_to_v6i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB17_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB17_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-NEXT: .LBB17_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB17_3
+; GFX9-NEXT: .LBB17_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB17_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB17_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s18, s18, 3
; GFX9-NEXT: s_add_i32 s17, s17, 3
; GFX9-NEXT: s_add_i32 s16, s16, 3
-; GFX9-NEXT: .LBB17_3: ; %end
+; GFX9-NEXT: .LBB17_5: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB17_4:
-; GFX9-NEXT: s_branch .LBB17_2
;
; GFX11-LABEL: bitcast_v3i32_to_v6i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB17_4
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB17_3
-; GFX11-NEXT: .LBB17_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB17_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB17_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: s_add_i32 s2, s2, 3
; GFX11-NEXT: s_add_i32 s1, s1, 3
; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: .LBB17_3: ; %end
+; GFX11-NEXT: .LBB17_4: ; %end
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB17_4:
-; GFX11-NEXT: s_branch .LBB17_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3374,7 +3570,7 @@ define inreg <3 x i32> @bitcast_v6i16_to_v3i32_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB19_4
+; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -3385,8 +3581,17 @@ define inreg <3 x i32> @bitcast_v6i16_to_v3i32_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_and_b32 s6, s18, 0xffff
; SI-NEXT: s_lshl_b32 s8, s7, 16
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_cbranch_execnz .LBB19_3
-; SI-NEXT: .LBB19_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB19_3
+; SI-NEXT: .LBB19_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB19_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB19_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -3402,23 +3607,28 @@ define inreg <3 x i32> @bitcast_v6i16_to_v3i32_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
-; SI-NEXT: .LBB19_3: ; %end
+; SI-NEXT: .LBB19_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB19_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB19_2
;
; VI-LABEL: bitcast_v6i16_to_v3i32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB19_4
+; VI-NEXT: s_cbranch_scc0 .LBB19_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB19_3
-; VI-NEXT: .LBB19_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB19_3
+; VI-NEXT: .LBB19_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB19_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB19_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s18, 3
; VI-NEXT: s_and_b32 s4, s18, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -3434,29 +3644,33 @@ define inreg <3 x i32> @bitcast_v6i16_to_v3i32_scalar(<6 x i16> inreg %a, i32 in
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB19_3: ; %end
+; VI-NEXT: .LBB19_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB19_4:
-; VI-NEXT: s_branch .LBB19_2
;
; GFX9-LABEL: bitcast_v6i16_to_v3i32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB19_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB19_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB19_4
-; GFX9-NEXT: .LBB19_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB19_3
+; GFX9-NEXT: .LBB19_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB19_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB19_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB19_3:
-; GFX9-NEXT: s_branch .LBB19_2
-; GFX9-NEXT: .LBB19_4:
+; GFX9-NEXT: .LBB19_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -3467,17 +3681,20 @@ define inreg <3 x i32> @bitcast_v6i16_to_v3i32_scalar(<6 x i16> inreg %a, i32 in
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB19_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB19_4
-; GFX11-NEXT: .LBB19_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB19_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB19_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB19_3:
-; GFX11-NEXT: s_branch .LBB19_2
; GFX11-NEXT: .LBB19_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -3773,7 +3990,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB21_3
+; SI-NEXT: s_cbranch_scc0 .LBB21_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s19, s17, 24
; SI-NEXT: s_lshr_b32 s22, s17, 16
@@ -3784,8 +4001,25 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; SI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB21_4
-; SI-NEXT: .LBB21_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[20:21], 0
+; SI-NEXT: s_branch .LBB21_3
+; SI-NEXT: .LBB21_2:
+; SI-NEXT: s_mov_b64 s[20:21], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr23
+; SI-NEXT: ; implicit-def: $sgpr22
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: .LBB21_3: ; %Flow
+; SI-NEXT: s_and_b64 s[20:21], s[20:21], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB21_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v8, s18, 1.0
; SI-NEXT: v_add_f32_e64 v17, s17, 1.0
; SI-NEXT: v_add_f32_e64 v16, s16, 1.0
@@ -3798,19 +4032,8 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v17
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v17
; SI-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; SI-NEXT: s_branch .LBB21_5
-; SI-NEXT: .LBB21_3:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr23
-; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: s_branch .LBB21_2
-; SI-NEXT: .LBB21_4:
+; SI-NEXT: s_branch .LBB21_6
+; SI-NEXT: .LBB21_5:
; SI-NEXT: v_mov_b32_e32 v16, s16
; SI-NEXT: v_mov_b32_e32 v17, s17
; SI-NEXT: v_mov_b32_e32 v8, s18
@@ -3823,7 +4046,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v14, s6
; SI-NEXT: v_mov_b32_e32 v1, s8
-; SI-NEXT: .LBB21_5: ; %end
+; SI-NEXT: .LBB21_6: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v16
; SI-NEXT: v_mov_b32_e32 v2, v14
; SI-NEXT: v_mov_b32_e32 v4, v17
@@ -3834,7 +4057,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB21_3
+; VI-NEXT: s_cbranch_scc0 .LBB21_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s19, s16, 8
; VI-NEXT: s_lshr_b32 s10, s18, 16
@@ -3845,8 +4068,25 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; VI-NEXT: s_lshr_b32 s14, s16, 16
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB21_4
-; VI-NEXT: .LBB21_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB21_3
+; VI-NEXT: .LBB21_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr19
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB21_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB21_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v8, s18, 1.0
; VI-NEXT: v_add_f32_e64 v14, s17, 1.0
; VI-NEXT: v_add_f32_e64 v13, s16, 1.0
@@ -3859,19 +4099,8 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; VI-NEXT: s_branch .LBB21_5
-; VI-NEXT: .LBB21_3:
-; VI-NEXT: ; implicit-def: $sgpr19
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB21_2
-; VI-NEXT: .LBB21_4:
+; VI-NEXT: s_branch .LBB21_6
+; VI-NEXT: .LBB21_5:
; VI-NEXT: v_mov_b32_e32 v13, s16
; VI-NEXT: v_mov_b32_e32 v14, s17
; VI-NEXT: v_mov_b32_e32 v8, s18
@@ -3884,7 +4113,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; VI-NEXT: v_mov_b32_e32 v10, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v3, s4
-; VI-NEXT: .LBB21_5: ; %end
+; VI-NEXT: .LBB21_6: ; %end
; VI-NEXT: v_mov_b32_e32 v0, v13
; VI-NEXT: v_mov_b32_e32 v4, v14
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -3893,7 +4122,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB21_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s19, s16, 8
; GFX9-NEXT: s_lshr_b32 s10, s18, 16
@@ -3904,8 +4133,25 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX9-NEXT: s_lshr_b32 s14, s16, 16
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB21_4
-; GFX9-NEXT: .LBB21_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB21_3
+; GFX9-NEXT: .LBB21_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr19
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB21_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v8, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v14, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v13, s16, 1.0
@@ -3918,19 +4164,8 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX9-NEXT: s_branch .LBB21_5
-; GFX9-NEXT: .LBB21_3:
-; GFX9-NEXT: ; implicit-def: $sgpr19
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB21_2
-; GFX9-NEXT: .LBB21_4:
+; GFX9-NEXT: s_branch .LBB21_6
+; GFX9-NEXT: .LBB21_5:
; GFX9-NEXT: v_mov_b32_e32 v13, s16
; GFX9-NEXT: v_mov_b32_e32 v14, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -3943,7 +4178,7 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX9-NEXT: v_mov_b32_e32 v10, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB21_5: ; %end
+; GFX9-NEXT: .LBB21_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v13
; GFX9-NEXT: v_mov_b32_e32 v4, v14
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3952,10 +4187,10 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB21_3
+; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s8, s2, 16
+; GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-NEXT: s_lshr_b32 s9, s2, 8
; GFX11-NEXT: s_lshr_b32 s10, s1, 24
; GFX11-NEXT: s_lshr_b32 s11, s1, 16
@@ -3964,9 +4199,25 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11-NEXT: s_lshr_b32 s14, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB21_4
-; GFX11-NEXT: .LBB21_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB21_3
+; GFX11-NEXT: .LBB21_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB21_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB21_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v8, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v14, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v13, s0, 1.0
@@ -3980,27 +4231,16 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX11-NEXT: s_branch .LBB21_5
-; GFX11-NEXT: .LBB21_3:
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB21_2
-; GFX11-NEXT: .LBB21_4:
+; GFX11-NEXT: s_branch .LBB21_6
+; GFX11-NEXT: .LBB21_5:
; GFX11-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB21_5: ; %end
+; GFX11-NEXT: .LBB21_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v13
; GFX11-NEXT: v_mov_b32_e32 v4, v14
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -4452,7 +4692,7 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB23_4
+; SI-NEXT: s_cbranch_scc0 .LBB23_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4481,8 +4721,17 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s7, s8, s7
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_cbranch_execnz .LBB23_3
-; SI-NEXT: .LBB23_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB23_3
+; SI-NEXT: .LBB23_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB23_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB23_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -4522,20 +4771,17 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; SI-NEXT: s_add_i32 s4, s4, 0x3000000
; SI-NEXT: s_add_i32 s5, s5, 0x3000000
; SI-NEXT: s_add_i32 s6, s6, 0x3000000
-; SI-NEXT: .LBB23_3: ; %end
+; SI-NEXT: .LBB23_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB23_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB23_2
;
; VI-LABEL: bitcast_v12i8_to_v3f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB23_4
+; VI-NEXT: s_cbranch_scc0 .LBB23_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -4556,8 +4802,17 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; VI-NEXT: v_perm_b32 v2, s26, v4, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: s_cbranch_execnz .LBB23_3
-; VI-NEXT: .LBB23_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB23_3
+; VI-NEXT: .LBB23_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; VI-NEXT: .LBB23_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB23_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -4589,17 +4844,14 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
-; VI-NEXT: .LBB23_3: ; %end
+; VI-NEXT: .LBB23_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB23_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; VI-NEXT: s_branch .LBB23_2
;
; GFX9-LABEL: bitcast_v12i8_to_v3f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB23_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -4620,8 +4872,17 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX9-NEXT: v_perm_b32 v2, s26, v4, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: s_cbranch_execnz .LBB23_3
-; GFX9-NEXT: .LBB23_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB23_3
+; GFX9-NEXT: .LBB23_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX9-NEXT: .LBB23_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -4651,18 +4912,15 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB23_3: ; %end
+; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB23_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX9-NEXT: s_branch .LBB23_2
;
; GFX11-LABEL: bitcast_v12i8_to_v3f32_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4680,9 +4938,17 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB23_3
-; GFX11-NEXT: .LBB23_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB23_3
+; GFX11-NEXT: .LBB23_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-NEXT: .LBB23_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -4712,11 +4978,8 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB23_3: ; %end
+; GFX11-NEXT: .LBB23_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB23_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-NEXT: s_branch .LBB23_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4851,7 +5114,7 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB25_3
+; SI-NEXT: s_cbranch_scc0 .LBB25_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s6, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s18, 16
@@ -4859,8 +5122,22 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; SI-NEXT: s_lshl_b32 s9, s17, 16
; SI-NEXT: s_and_b32 s10, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s16, 16
-; SI-NEXT: s_cbranch_execnz .LBB25_4
-; SI-NEXT: .LBB25_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB25_3
+; SI-NEXT: .LBB25_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: .LBB25_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB25_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
@@ -4870,23 +5147,15 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_branch .LBB25_5
-; SI-NEXT: .LBB25_3:
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: s_branch .LBB25_2
-; SI-NEXT: .LBB25_4:
+; SI-NEXT: s_branch .LBB25_6
+; SI-NEXT: .LBB25_5:
; SI-NEXT: v_mov_b32_e32 v0, s11
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v2, s9
; SI-NEXT: v_mov_b32_e32 v5, s8
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: v_mov_b32_e32 v4, s6
-; SI-NEXT: .LBB25_5: ; %end
+; SI-NEXT: .LBB25_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -4905,17 +5174,23 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB25_3
+; VI-NEXT: s_cbranch_scc0 .LBB25_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB25_4
-; VI-NEXT: .LBB25_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB25_3
+; VI-NEXT: .LBB25_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB25_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB25_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB25_3:
-; VI-NEXT: s_branch .LBB25_2
-; VI-NEXT: .LBB25_4:
+; VI-NEXT: .LBB25_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -4926,17 +5201,23 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB25_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB25_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB25_4
-; GFX9-NEXT: .LBB25_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB25_3
+; GFX9-NEXT: .LBB25_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB25_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB25_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB25_3:
-; GFX9-NEXT: s_branch .LBB25_2
-; GFX9-NEXT: .LBB25_4:
+; GFX9-NEXT: .LBB25_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -4948,17 +5229,20 @@ define inreg <6 x bfloat> @bitcast_v3f32_to_v6bf16_scalar(<3 x float> inreg %a,
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB25_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB25_4
-; GFX11-NEXT: .LBB25_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB25_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB25_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB25_3:
-; GFX11-NEXT: s_branch .LBB25_2
; GFX11-NEXT: .LBB25_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -5344,7 +5628,7 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB27_4
+; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
; SI-NEXT: v_lshr_b64 v[0:1], v[8:9], 16
@@ -5352,8 +5636,17 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; SI-NEXT: v_lshr_b64 v[1:2], v[6:7], 16
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v10
; SI-NEXT: v_lshr_b64 v[2:3], v[4:5], 16
-; SI-NEXT: s_cbranch_execnz .LBB27_3
-; SI-NEXT: .LBB27_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB27_3
+; SI-NEXT: .LBB27_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; SI-NEXT: .LBB27_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB27_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v12
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
@@ -5372,20 +5665,25 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: .LBB27_3: ; %end
+; SI-NEXT: .LBB27_5: ; %end
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB27_4:
-; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; SI-NEXT: s_branch .LBB27_2
;
; VI-LABEL: bitcast_v6bf16_to_v3f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB27_3
+; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB27_4
-; VI-NEXT: .LBB27_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB27_3
+; VI-NEXT: .LBB27_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB27_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB27_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s18, 16
; VI-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v5
@@ -5443,9 +5741,7 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB27_3:
-; VI-NEXT: s_branch .LBB27_2
-; VI-NEXT: .LBB27_4:
+; VI-NEXT: .LBB27_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5455,10 +5751,18 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB27_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB27_4
-; GFX9-NEXT: .LBB27_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB27_3
+; GFX9-NEXT: .LBB27_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB27_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s18
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -5519,9 +5823,7 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB27_3:
-; GFX9-NEXT: s_branch .LBB27_2
-; GFX9-NEXT: .LBB27_4:
+; GFX9-NEXT: .LBB27_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5532,11 +5834,16 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-TRUE16-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: .LBB27_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -5602,8 +5909,6 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB27_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB27_2
; GFX11-TRUE16-NEXT: .LBB27_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s2
@@ -5614,11 +5919,16 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB27_4
-; GFX11-FAKE16-NEXT: .LBB27_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: .LBB27_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -5686,8 +5996,6 @@ define inreg <3 x float> @bitcast_v6bf16_to_v3f32_scalar(<6 x bfloat> inreg %a,
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB27_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB27_2
; GFX11-FAKE16-NEXT: .LBB27_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s2
@@ -5811,33 +6119,39 @@ define inreg <6 x half> @bitcast_v3f32_to_v6f16_scalar(<3 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB29_3
+; SI-NEXT: s_cbranch_scc0 .LBB29_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB29_4
-; SI-NEXT: .LBB29_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB29_3
+; SI-NEXT: .LBB29_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB29_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB29_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[3:4], v[2:3], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; SI-NEXT: s_branch .LBB29_5
-; SI-NEXT: .LBB29_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB29_2
-; SI-NEXT: .LBB29_4:
+; SI-NEXT: s_branch .LBB29_6
+; SI-NEXT: .LBB29_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v4, s10
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v5, s6
-; SI-NEXT: .LBB29_5: ; %end
+; SI-NEXT: .LBB29_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -5853,17 +6167,23 @@ define inreg <6 x half> @bitcast_v3f32_to_v6f16_scalar(<3 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB29_3
+; VI-NEXT: s_cbranch_scc0 .LBB29_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB29_4
-; VI-NEXT: .LBB29_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB29_3
+; VI-NEXT: .LBB29_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB29_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB29_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB29_3:
-; VI-NEXT: s_branch .LBB29_2
-; VI-NEXT: .LBB29_4:
+; VI-NEXT: .LBB29_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -5874,17 +6194,23 @@ define inreg <6 x half> @bitcast_v3f32_to_v6f16_scalar(<3 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB29_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB29_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB29_4
-; GFX9-NEXT: .LBB29_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB29_3
+; GFX9-NEXT: .LBB29_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB29_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB29_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB29_3:
-; GFX9-NEXT: s_branch .LBB29_2
-; GFX9-NEXT: .LBB29_4:
+; GFX9-NEXT: .LBB29_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -5896,17 +6222,20 @@ define inreg <6 x half> @bitcast_v3f32_to_v6f16_scalar(<3 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB29_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB29_4
-; GFX11-NEXT: .LBB29_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB29_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB29_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB29_3:
-; GFX11-NEXT: s_branch .LBB29_2
; GFX11-NEXT: .LBB29_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6074,7 +6403,7 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB31_3
+; SI-NEXT: s_cbranch_scc0 .LBB31_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -6085,8 +6414,17 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; SI-NEXT: s_and_b32 s6, s18, 0xffff
; SI-NEXT: s_lshl_b32 s8, s7, 16
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_cbranch_execnz .LBB31_4
-; SI-NEXT: .LBB31_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB31_3
+; SI-NEXT: .LBB31_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB31_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB31_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s11
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s10
@@ -6112,10 +6450,7 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB31_3:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB31_2
-; SI-NEXT: .LBB31_4:
+; SI-NEXT: .LBB31_5:
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -6125,10 +6460,18 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB31_3
+; VI-NEXT: s_cbranch_scc0 .LBB31_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB31_4
-; VI-NEXT: .LBB31_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB31_3
+; VI-NEXT: .LBB31_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB31_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB31_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s18, 16
; VI-NEXT: v_mov_b32_e32 v0, 0x200
; VI-NEXT: v_mov_b32_e32 v1, s4
@@ -6146,9 +6489,7 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; VI-NEXT: v_add_f16_e32 v0, s16, v0
; VI-NEXT: v_or_b32_e32 v0, v0, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB31_3:
-; VI-NEXT: s_branch .LBB31_2
-; VI-NEXT: .LBB31_4:
+; VI-NEXT: .LBB31_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6158,18 +6499,24 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB31_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB31_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB31_4
-; GFX9-NEXT: .LBB31_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB31_3
+; GFX9-NEXT: .LBB31_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB31_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB31_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB31_3:
-; GFX9-NEXT: s_branch .LBB31_2
-; GFX9-NEXT: .LBB31_4:
+; GFX9-NEXT: .LBB31_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6180,17 +6527,20 @@ define inreg <3 x float> @bitcast_v6f16_to_v3f32_scalar(<6 x half> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB31_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB31_4
-; GFX11-NEXT: .LBB31_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB31_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB31_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB31_3:
-; GFX11-NEXT: s_branch .LBB31_2
; GFX11-NEXT: .LBB31_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -6314,33 +6664,39 @@ define inreg <6 x i16> @bitcast_v3f32_to_v6i16_scalar(<3 x float> inreg %a, i32
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB33_3
+; SI-NEXT: s_cbranch_scc0 .LBB33_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b64 s[4:5], s[18:19], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 16
-; SI-NEXT: s_cbranch_execnz .LBB33_4
-; SI-NEXT: .LBB33_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB33_3
+; SI-NEXT: .LBB33_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: .LBB33_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, 1
+; SI-NEXT: s_cbranch_scc1 .LBB33_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_add_f32_e64 v2, s18, 1.0
; SI-NEXT: v_add_f32_e64 v1, s17, 1.0
; SI-NEXT: v_add_f32_e64 v0, s16, 1.0
; SI-NEXT: v_lshr_b64 v[5:6], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[3:4], v[2:3], 16
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; SI-NEXT: s_branch .LBB33_5
-; SI-NEXT: .LBB33_3:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_branch .LBB33_2
-; SI-NEXT: .LBB33_4:
+; SI-NEXT: s_branch .LBB33_6
+; SI-NEXT: .LBB33_5:
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
; SI-NEXT: v_mov_b32_e32 v4, s10
; SI-NEXT: v_mov_b32_e32 v3, s4
; SI-NEXT: v_mov_b32_e32 v5, s6
-; SI-NEXT: .LBB33_5: ; %end
+; SI-NEXT: .LBB33_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -6356,17 +6712,23 @@ define inreg <6 x i16> @bitcast_v3f32_to_v6i16_scalar(<3 x float> inreg %a, i32
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB33_3
+; VI-NEXT: s_cbranch_scc0 .LBB33_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB33_4
-; VI-NEXT: .LBB33_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB33_3
+; VI-NEXT: .LBB33_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB33_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB33_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: v_add_f32_e64 v2, s18, 1.0
; VI-NEXT: v_add_f32_e64 v1, s17, 1.0
; VI-NEXT: v_add_f32_e64 v0, s16, 1.0
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB33_3:
-; VI-NEXT: s_branch .LBB33_2
-; VI-NEXT: .LBB33_4:
+; VI-NEXT: .LBB33_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -6377,17 +6739,23 @@ define inreg <6 x i16> @bitcast_v3f32_to_v6i16_scalar(<3 x float> inreg %a, i32
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB33_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB33_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB33_4
-; GFX9-NEXT: .LBB33_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB33_3
+; GFX9-NEXT: .LBB33_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB33_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB33_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_add_f32_e64 v2, s18, 1.0
; GFX9-NEXT: v_add_f32_e64 v1, s17, 1.0
; GFX9-NEXT: v_add_f32_e64 v0, s16, 1.0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB33_3:
-; GFX9-NEXT: s_branch .LBB33_2
-; GFX9-NEXT: .LBB33_4:
+; GFX9-NEXT: .LBB33_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6399,17 +6767,20 @@ define inreg <6 x i16> @bitcast_v3f32_to_v6i16_scalar(<3 x float> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB33_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB33_4
-; GFX11-NEXT: .LBB33_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB33_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB33_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX11-NEXT: v_add_f32_e64 v1, s1, 1.0
; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB33_3:
-; GFX11-NEXT: s_branch .LBB33_2
; GFX11-NEXT: .LBB33_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -6564,7 +6935,7 @@ define inreg <3 x float> @bitcast_v6i16_to_v3f32_scalar(<6 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s10, s17, 16
; SI-NEXT: s_lshr_b32 s11, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB35_4
+; SI-NEXT: s_cbranch_scc0 .LBB35_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -6575,8 +6946,17 @@ define inreg <3 x float> @bitcast_v6i16_to_v3f32_scalar(<6 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s6, s18, 0xffff
; SI-NEXT: s_lshl_b32 s8, s7, 16
; SI-NEXT: s_or_b32 s6, s6, s8
-; SI-NEXT: s_cbranch_execnz .LBB35_3
-; SI-NEXT: .LBB35_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_branch .LBB35_3
+; SI-NEXT: .LBB35_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
+; SI-NEXT: .LBB35_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB35_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s11, 16
@@ -6592,23 +6972,28 @@ define inreg <3 x float> @bitcast_v6i16_to_v3f32_scalar(<6 x i16> inreg %a, i32
; SI-NEXT: s_add_i32 s4, s4, 0x30000
; SI-NEXT: s_add_i32 s5, s5, 0x30000
; SI-NEXT: s_add_i32 s6, s6, 0x30000
-; SI-NEXT: .LBB35_3: ; %end
+; SI-NEXT: .LBB35_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB35_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6
-; SI-NEXT: s_branch .LBB35_2
;
; VI-LABEL: bitcast_v6i16_to_v3f32_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB35_4
+; VI-NEXT: s_cbranch_scc0 .LBB35_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB35_3
-; VI-NEXT: .LBB35_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB35_3
+; VI-NEXT: .LBB35_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB35_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB35_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s18, 3
; VI-NEXT: s_and_b32 s4, s18, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -6624,29 +7009,33 @@ define inreg <3 x float> @bitcast_v6i16_to_v3f32_scalar(<6 x i16> inreg %a, i32
; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB35_3: ; %end
+; VI-NEXT: .LBB35_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB35_4:
-; VI-NEXT: s_branch .LBB35_2
;
; GFX9-LABEL: bitcast_v6i16_to_v3f32_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB35_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB35_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB35_4
-; GFX9-NEXT: .LBB35_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB35_3
+; GFX9-NEXT: .LBB35_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB35_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB35_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB35_3:
-; GFX9-NEXT: s_branch .LBB35_2
-; GFX9-NEXT: .LBB35_4:
+; GFX9-NEXT: .LBB35_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -6657,17 +7046,20 @@ define inreg <3 x float> @bitcast_v6i16_to_v3f32_scalar(<6 x i16> inreg %a, i32
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB35_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB35_4
-; GFX11-NEXT: .LBB35_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: .LBB35_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB35_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB35_3:
-; GFX11-NEXT: s_branch .LBB35_2
; GFX11-NEXT: .LBB35_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -7139,7 +7531,7 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB37_4
+; SI-NEXT: s_cbranch_scc0 .LBB37_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
@@ -7165,8 +7557,22 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s27, 24
; SI-NEXT: s_or_b32 s11, s5, s4
-; SI-NEXT: s_cbranch_execnz .LBB37_3
-; SI-NEXT: .LBB37_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB37_3
+; SI-NEXT: .LBB37_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB37_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB37_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s24, s24, 3
; SI-NEXT: s_and_b32 s4, s24, 0xff
; SI-NEXT: s_lshl_b32 s5, s25, 8
@@ -7212,7 +7618,7 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s11, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s4, 16
-; SI-NEXT: .LBB37_3: ; %end
+; SI-NEXT: .LBB37_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
@@ -7226,20 +7632,12 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s9
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB37_4:
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB37_2
;
; VI-LABEL: bitcast_v12i8_to_v6bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB37_4
+; VI-NEXT: s_cbranch_scc0 .LBB37_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -7260,8 +7658,17 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; VI-NEXT: v_perm_b32 v2, s26, v4, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: s_cbranch_execnz .LBB37_3
-; VI-NEXT: .LBB37_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB37_3
+; VI-NEXT: .LBB37_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB37_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB37_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -7293,17 +7700,14 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
-; VI-NEXT: .LBB37_3: ; %end
+; VI-NEXT: .LBB37_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB37_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB37_2
;
; GFX9-LABEL: bitcast_v12i8_to_v6bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB37_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -7324,8 +7728,17 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v2, s26, v4, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: s_cbranch_execnz .LBB37_3
-; GFX9-NEXT: .LBB37_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB37_3
+; GFX9-NEXT: .LBB37_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB37_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -7355,18 +7768,15 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB37_3: ; %end
+; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB37_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB37_2
;
; GFX11-LABEL: bitcast_v12i8_to_v6bf16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB37_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -7384,9 +7794,17 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB37_3
-; GFX11-NEXT: .LBB37_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB37_3
+; GFX11-NEXT: .LBB37_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB37_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -7416,11 +7834,8 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB37_3: ; %end
+; GFX11-NEXT: .LBB37_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB37_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB37_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7978,7 +8393,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s7
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB39_4
+; SI-NEXT: s_cbranch_scc0 .LBB39_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
@@ -7991,18 +8406,35 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v19
; SI-NEXT: v_lshrrev_b32_e32 v8, 8, v17
; SI-NEXT: v_lshrrev_b32_e32 v12, 8, v13
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: v_lshr_b64 v[14:15], v[16:17], 16
; SI-NEXT: v_lshr_b64 v[1:2], v[16:17], 8
-; SI-NEXT: s_cbranch_execnz .LBB39_3
-; SI-NEXT: .LBB39_2: ; %cmp.true
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
+; SI-NEXT: s_branch .LBB39_3
+; SI-NEXT: .LBB39_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr16
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr13
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: .LBB39_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB39_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshr_b64 v[16:17], v[0:1], 16
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v9
@@ -8020,7 +8452,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v12, 8, v13
; SI-NEXT: v_lshrrev_b32_e32 v7, 24, v0
; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v5
-; SI-NEXT: .LBB39_3: ; %end
+; SI-NEXT: .LBB39_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v16
; SI-NEXT: v_mov_b32_e32 v2, v14
; SI-NEXT: v_mov_b32_e32 v4, v17
@@ -8028,23 +8460,12 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v8, v13
; SI-NEXT: v_mov_b32_e32 v9, v12
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB39_4:
-; SI-NEXT: ; implicit-def: $vgpr16
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr11
-; SI-NEXT: s_branch .LBB39_2
;
; VI-LABEL: bitcast_v6bf16_to_v12i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB39_3
+; VI-NEXT: s_cbranch_scc0 .LBB39_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s19, s18, 16
; VI-NEXT: s_lshr_b32 s15, s18, 8
@@ -8055,8 +8476,25 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; VI-NEXT: s_lshr_b32 s13, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB39_4
-; VI-NEXT: .LBB39_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB39_3
+; VI-NEXT: .LBB39_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr19
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB39_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB39_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v3, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v3
@@ -8123,19 +8561,8 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; VI-NEXT: s_branch .LBB39_5
-; VI-NEXT: .LBB39_3:
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr19
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB39_2
-; VI-NEXT: .LBB39_4:
+; VI-NEXT: s_branch .LBB39_6
+; VI-NEXT: .LBB39_5:
; VI-NEXT: v_mov_b32_e32 v8, s18
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v4, s17
@@ -8148,7 +8575,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v5, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: v_mov_b32_e32 v14, s4
-; VI-NEXT: .LBB39_5: ; %end
+; VI-NEXT: .LBB39_6: ; %end
; VI-NEXT: v_mov_b32_e32 v3, v14
; VI-NEXT: v_mov_b32_e32 v9, v13
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -8157,7 +8584,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s19, s18, 16
; GFX9-NEXT: s_lshr_b32 s12, s18, 8
@@ -8168,8 +8595,25 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: s_lshr_b32 s13, s16, 8
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB39_4
-; GFX9-NEXT: .LBB39_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB39_3
+; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr19
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB39_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
@@ -8240,18 +8684,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX9-NEXT: v_mov_b32_e32 v4, v13
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB39_3:
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr19
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB39_2
-; GFX9-NEXT: .LBB39_4:
+; GFX9-NEXT: .LBB39_5:
; GFX9-NEXT: v_mov_b32_e32 v8, s18
; GFX9-NEXT: v_mov_b32_e32 v10, s19
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -8271,7 +8704,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s2, 8
@@ -8282,9 +8715,25 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-TRUE16-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_branch .LBB39_3
+; GFX11-TRUE16-NEXT: .LBB39_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: .LBB39_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -8363,18 +8812,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB39_3:
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: s_branch .LBB39_2
-; GFX11-TRUE16-NEXT: .LBB39_4:
+; GFX11-TRUE16-NEXT: .LBB39_5:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s10
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v1, s11
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s8
@@ -8388,7 +8826,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_3
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: s_lshr_b32 s14, s2, 16
; GFX11-FAKE16-NEXT: s_lshr_b32 s10, s2, 8
@@ -8399,9 +8837,25 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_lshr_b32 s11, s0, 8
; GFX11-FAKE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-FAKE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB39_4
-; GFX11-FAKE16-NEXT: .LBB39_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_branch .LBB39_3
+; GFX11-FAKE16-NEXT: .LBB39_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-FAKE16-NEXT: .LBB39_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB39_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s1
; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -8479,18 +8933,7 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v13
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB39_3:
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-FAKE16-NEXT: s_branch .LBB39_2
-; GFX11-FAKE16-NEXT: .LBB39_4:
+; GFX11-FAKE16-NEXT: .LBB39_5:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s10
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v1, s11
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s8
@@ -8963,7 +9406,7 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB41_4
+; SI-NEXT: s_cbranch_scc0 .LBB41_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -8995,9 +9438,22 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; SI-NEXT: s_or_b32 s4, s4, s12
; SI-NEXT: s_lshr_b32 s7, s9, 16
; SI-NEXT: s_lshr_b32 s11, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s13
-; SI-NEXT: s_cbranch_execnz .LBB41_3
-; SI-NEXT: .LBB41_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB41_3
+; SI-NEXT: .LBB41_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB41_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB41_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -9040,7 +9496,7 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s7, s5, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
-; SI-NEXT: .LBB41_3: ; %end
+; SI-NEXT: .LBB41_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -9054,19 +9510,12 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB41_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB41_2
;
; VI-LABEL: bitcast_v12i8_to_v6f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB41_4
+; VI-NEXT: s_cbranch_scc0 .LBB41_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -9087,8 +9536,17 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; VI-NEXT: v_perm_b32 v2, s26, v4, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: s_cbranch_execnz .LBB41_3
-; VI-NEXT: .LBB41_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB41_3
+; VI-NEXT: .LBB41_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB41_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB41_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -9120,17 +9578,14 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
-; VI-NEXT: .LBB41_3: ; %end
+; VI-NEXT: .LBB41_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB41_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB41_2
;
; GFX9-LABEL: bitcast_v12i8_to_v6f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB41_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -9151,8 +9606,17 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX9-NEXT: v_perm_b32 v2, s26, v4, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: s_cbranch_execnz .LBB41_3
-; GFX9-NEXT: .LBB41_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB41_3
+; GFX9-NEXT: .LBB41_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB41_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -9182,18 +9646,15 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB41_3: ; %end
+; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB41_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB41_2
;
; GFX11-LABEL: bitcast_v12i8_to_v6f16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9211,9 +9672,17 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB41_3
-; GFX11-NEXT: .LBB41_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -9243,11 +9712,8 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB41_3: ; %end
+; GFX11-NEXT: .LBB41_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB41_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9590,7 +10056,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s15, s17, 16
; SI-NEXT: s_lshr_b32 s20, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB43_3
+; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s20, 16
@@ -9608,8 +10074,25 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s9, s11, 8
; SI-NEXT: s_bfe_u32 s19, s15, 0x80008
; SI-NEXT: s_bfe_u32 s21, s14, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB43_4
-; SI-NEXT: .LBB43_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB43_3
+; SI-NEXT: .LBB43_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: .LBB43_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB43_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s20
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s15
@@ -9641,19 +10124,8 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; SI-NEXT: v_bfe_u32 v7, v6, 8, 8
; SI-NEXT: v_bfe_u32 v11, v10, 8, 8
-; SI-NEXT: s_branch .LBB43_5
-; SI-NEXT: .LBB43_3:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: s_branch .LBB43_2
-; SI-NEXT: .LBB43_4:
+; SI-NEXT: s_branch .LBB43_6
+; SI-NEXT: .LBB43_5:
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: v_mov_b32_e32 v6, s15
; SI-NEXT: v_mov_b32_e32 v11, s21
@@ -9666,7 +10138,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v3, s6
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v1, s10
-; SI-NEXT: .LBB43_5: ; %end
+; SI-NEXT: .LBB43_6: ; %end
; SI-NEXT: v_mov_b32_e32 v0, v12
; SI-NEXT: v_mov_b32_e32 v2, v4
; SI-NEXT: v_mov_b32_e32 v4, v13
@@ -9676,7 +10148,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB43_3
+; VI-NEXT: s_cbranch_scc0 .LBB43_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s19, s16, 16
; VI-NEXT: s_lshr_b32 s14, s18, 16
@@ -9687,8 +10159,25 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; VI-NEXT: s_lshr_b32 s12, s16, 8
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB43_4
-; VI-NEXT: .LBB43_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB43_3
+; VI-NEXT: .LBB43_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr19
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB43_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB43_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s17, 16
; VI-NEXT: v_mov_b32_e32 v1, 0x200
; VI-NEXT: v_add_f16_e32 v6, s4, v1
@@ -9714,18 +10203,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; VI-NEXT: v_bfe_u32 v7, v6, 8, 8
; VI-NEXT: v_mov_b32_e32 v4, v13
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB43_3:
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr19
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB43_2
-; VI-NEXT: .LBB43_4:
+; VI-NEXT: .LBB43_5:
; VI-NEXT: v_mov_b32_e32 v2, s19
; VI-NEXT: v_mov_b32_e32 v6, s15
; VI-NEXT: v_mov_b32_e32 v10, s14
@@ -9744,7 +10222,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB43_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s19, s16, 8
; GFX9-NEXT: s_lshr_b32 s10, s18, 16
@@ -9755,8 +10233,25 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX9-NEXT: s_lshr_b32 s14, s16, 16
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB43_4
-; GFX9-NEXT: .LBB43_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB43_3
+; GFX9-NEXT: .LBB43_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr19
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB43_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v15, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v14, s16, v0 op_sel_hi:[1,0]
@@ -9771,19 +10266,8 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v15
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v14
-; GFX9-NEXT: s_branch .LBB43_5
-; GFX9-NEXT: .LBB43_3:
-; GFX9-NEXT: ; implicit-def: $sgpr19
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB43_2
-; GFX9-NEXT: .LBB43_4:
+; GFX9-NEXT: s_branch .LBB43_6
+; GFX9-NEXT: .LBB43_5:
; GFX9-NEXT: v_mov_b32_e32 v14, s16
; GFX9-NEXT: v_mov_b32_e32 v15, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -9796,7 +10280,7 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX9-NEXT: v_mov_b32_e32 v10, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB43_5: ; %end
+; GFX9-NEXT: .LBB43_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v14
; GFX9-NEXT: v_mov_b32_e32 v4, v15
; GFX9-NEXT: v_mov_b32_e32 v9, v13
@@ -9806,10 +10290,10 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_3
+; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB43_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s8, s2, 16
+; GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-NEXT: s_lshr_b32 s9, s2, 8
; GFX11-NEXT: s_lshr_b32 s10, s1, 24
; GFX11-NEXT: s_lshr_b32 s11, s1, 16
@@ -9818,9 +10302,25 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX11-NEXT: s_lshr_b32 s14, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB43_4
-; GFX11-NEXT: .LBB43_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB43_3
+; GFX11-NEXT: .LBB43_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB43_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v15, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v14, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
@@ -9836,27 +10336,16 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v15
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v14
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v14
-; GFX11-NEXT: s_branch .LBB43_5
-; GFX11-NEXT: .LBB43_3:
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB43_2
-; GFX11-NEXT: .LBB43_4:
+; GFX11-NEXT: s_branch .LBB43_6
+; GFX11-NEXT: .LBB43_5:
; GFX11-NEXT: v_dual_mov_b32 v14, s0 :: v_dual_mov_b32 v15, s1
; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v13, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v13, s9 :: v_dual_mov_b32 v10, s3
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB43_5: ; %end
+; GFX11-NEXT: .LBB43_6: ; %end
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_dual_mov_b32 v0, v14 :: v_dual_mov_b32 v9, v13
; GFX11-NEXT: v_mov_b32_e32 v4, v15
@@ -10326,7 +10815,7 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s28, 0
-; SI-NEXT: s_cbranch_scc0 .LBB45_4
+; SI-NEXT: s_cbranch_scc0 .LBB45_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -10358,9 +10847,22 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s4, s4, s12
; SI-NEXT: s_lshr_b32 s7, s9, 16
; SI-NEXT: s_lshr_b32 s11, s8, 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s13
-; SI-NEXT: s_cbranch_execnz .LBB45_3
-; SI-NEXT: .LBB45_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB45_3
+; SI-NEXT: .LBB45_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB45_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s8, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cbranch_scc1 .LBB45_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xff
; SI-NEXT: s_lshl_b32 s5, s17, 8
@@ -10403,7 +10905,7 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s7, s5, 16
; SI-NEXT: s_lshr_b32 s11, s10, 16
-; SI-NEXT: .LBB45_3: ; %end
+; SI-NEXT: .LBB45_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -10417,19 +10919,12 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB45_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB45_2
;
; VI-LABEL: bitcast_v12i8_to_v6i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s28, 0
-; VI-NEXT: s_cbranch_scc0 .LBB45_4
+; VI-NEXT: s_cbranch_scc0 .LBB45_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; VI-NEXT: v_mov_b32_e32 v1, s19
@@ -10450,8 +10945,17 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; VI-NEXT: v_perm_b32 v2, s26, v4, v2
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: s_cbranch_execnz .LBB45_3
-; VI-NEXT: .LBB45_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB45_3
+; VI-NEXT: .LBB45_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; VI-NEXT: .LBB45_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB45_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s16, s16, 3
; VI-NEXT: v_mov_b32_e32 v0, s17
; VI-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -10483,17 +10987,14 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3000000, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x3000000, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x3000000, v2
-; VI-NEXT: .LBB45_3: ; %end
+; VI-NEXT: .LBB45_5: ; %end
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB45_4:
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; VI-NEXT: s_branch .LBB45_2
;
; GFX9-LABEL: bitcast_v12i8_to_v6i16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s28, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX9-NEXT: s_cbranch_scc0 .LBB45_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX9-NEXT: v_mov_b32_e32 v1, s19
@@ -10514,8 +11015,17 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: v_perm_b32 v2, s26, v4, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX9-NEXT: s_cbranch_execnz .LBB45_3
-; GFX9-NEXT: .LBB45_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB45_3
+; GFX9-NEXT: .LBB45_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX9-NEXT: .LBB45_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_add_i32 s16, s16, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s17
; GFX9-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -10545,18 +11055,15 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: v_add_u32_e32 v3, 0x300, v3
; GFX9-NEXT: v_add_u32_sdwa v2, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT: .LBB45_3: ; %end
+; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB45_4:
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX9-NEXT: s_branch .LBB45_2
;
; GFX11-LABEL: bitcast_v12i8_to_v6i16_scalar:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s24, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_4
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10574,9 +11081,17 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB45_3
-; GFX11-NEXT: .LBB45_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
; GFX11-NEXT: s_add_i32 s0, s0, 3
; GFX11-NEXT: s_add_i32 s2, s2, 3
@@ -10606,11 +11121,8 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB45_3: ; %end
+; GFX11-NEXT: .LBB45_5: ; %end
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB45_4:
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: s_branch .LBB45_2
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10946,7 +11458,7 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s15, s17, 16
; SI-NEXT: s_lshr_b32 s20, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB47_4
+; SI-NEXT: s_cbranch_scc0 .LBB47_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s20, 16
@@ -10964,8 +11476,25 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s21, s19, 8
; SI-NEXT: s_bfe_u32 s9, s15, 0x80008
; SI-NEXT: s_bfe_u32 s11, s14, 0x80008
-; SI-NEXT: s_cbranch_execnz .LBB47_3
-; SI-NEXT: .LBB47_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[12:13], 0
+; SI-NEXT: s_branch .LBB47_3
+; SI-NEXT: .LBB47_2:
+; SI-NEXT: s_mov_b64 s[12:13], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr19
+; SI-NEXT: ; implicit-def: $sgpr21
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: .LBB47_3: ; %Flow
+; SI-NEXT: s_and_b64 s[12:13], s[12:13], exec
+; SI-NEXT: s_cselect_b32 s12, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s12, 1
+; SI-NEXT: s_cbranch_scc1 .LBB47_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s20, 16
@@ -10990,7 +11519,7 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: s_lshr_b32 s11, s19, 24
; SI-NEXT: s_lshr_b32 s14, s19, 16
; SI-NEXT: s_lshr_b32 s21, s19, 8
-; SI-NEXT: .LBB47_3: ; %end
+; SI-NEXT: .LBB47_5: ; %end
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v2, s8
@@ -11004,23 +11533,12 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; SI-NEXT: v_mov_b32_e32 v10, s14
; SI-NEXT: v_mov_b32_e32 v11, s11
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB47_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr19
-; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: s_branch .LBB47_2
;
; VI-LABEL: bitcast_v6i16_to_v12i8_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB47_4
+; VI-NEXT: s_cbranch_scc0 .LBB47_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: s_lshr_b32 s19, s16, 8
; VI-NEXT: s_lshr_b32 s10, s18, 16
@@ -11031,8 +11549,25 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s15, s16, 16
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; VI-NEXT: s_cbranch_execnz .LBB47_3
-; VI-NEXT: .LBB47_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[8:9], 0
+; VI-NEXT: s_branch .LBB47_3
+; VI-NEXT: .LBB47_2:
+; VI-NEXT: s_mov_b64 s[8:9], -1
+; VI-NEXT: ; implicit-def: $sgpr19
+; VI-NEXT: ; implicit-def: $sgpr15
+; VI-NEXT: ; implicit-def: $sgpr4
+; VI-NEXT: ; implicit-def: $sgpr14
+; VI-NEXT: ; implicit-def: $sgpr13
+; VI-NEXT: ; implicit-def: $sgpr12
+; VI-NEXT: ; implicit-def: $sgpr11
+; VI-NEXT: ; implicit-def: $sgpr10
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: .LBB47_3: ; %Flow
+; VI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; VI-NEXT: s_cselect_b32 s5, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s5, 1
+; VI-NEXT: s_cbranch_scc1 .LBB47_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s17, 3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: s_and_b32 s5, s5, 0xffff
@@ -11057,7 +11592,7 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; VI-NEXT: s_lshr_b32 s14, s17, 8
; VI-NEXT: s_lshr_b32 s15, s16, 16
; VI-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
-; VI-NEXT: .LBB47_3: ; %end
+; VI-NEXT: .LBB47_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s19
; VI-NEXT: v_mov_b32_e32 v2, s15
@@ -11071,23 +11606,12 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v10, s10
; VI-NEXT: v_mov_b32_e32 v11, s6
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB47_4:
-; VI-NEXT: ; implicit-def: $sgpr19
-; VI-NEXT: ; implicit-def: $sgpr15
-; VI-NEXT: ; implicit-def: $sgpr4
-; VI-NEXT: ; implicit-def: $sgpr14
-; VI-NEXT: ; implicit-def: $sgpr13
-; VI-NEXT: ; implicit-def: $sgpr12
-; VI-NEXT: ; implicit-def: $sgpr11
-; VI-NEXT: ; implicit-def: $sgpr10
-; VI-NEXT: ; implicit-def: $sgpr6
-; VI-NEXT: s_branch .LBB47_2
;
; GFX9-LABEL: bitcast_v6i16_to_v12i8_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB47_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: s_lshr_b32 s19, s16, 8
; GFX9-NEXT: s_lshr_b32 s10, s18, 16
@@ -11098,8 +11622,25 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b32 s14, s16, 16
; GFX9-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
-; GFX9-NEXT: s_cbranch_execnz .LBB47_4
-; GFX9-NEXT: .LBB47_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_branch .LBB47_3
+; GFX9-NEXT: .LBB47_2:
+; GFX9-NEXT: s_mov_b64 s[8:9], -1
+; GFX9-NEXT: ; implicit-def: $sgpr19
+; GFX9-NEXT: ; implicit-def: $sgpr14
+; GFX9-NEXT: ; implicit-def: $sgpr4
+; GFX9-NEXT: ; implicit-def: $sgpr15
+; GFX9-NEXT: ; implicit-def: $sgpr13
+; GFX9-NEXT: ; implicit-def: $sgpr12
+; GFX9-NEXT: ; implicit-def: $sgpr11
+; GFX9-NEXT: ; implicit-def: $sgpr10
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: .LBB47_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s5, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v8, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v14, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v13, s16, 3 op_sel_hi:[1,0]
@@ -11112,19 +11653,8 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX9-NEXT: s_branch .LBB47_5
-; GFX9-NEXT: .LBB47_3:
-; GFX9-NEXT: ; implicit-def: $sgpr19
-; GFX9-NEXT: ; implicit-def: $sgpr14
-; GFX9-NEXT: ; implicit-def: $sgpr4
-; GFX9-NEXT: ; implicit-def: $sgpr15
-; GFX9-NEXT: ; implicit-def: $sgpr13
-; GFX9-NEXT: ; implicit-def: $sgpr12
-; GFX9-NEXT: ; implicit-def: $sgpr11
-; GFX9-NEXT: ; implicit-def: $sgpr10
-; GFX9-NEXT: ; implicit-def: $sgpr6
-; GFX9-NEXT: s_branch .LBB47_2
-; GFX9-NEXT: .LBB47_4:
+; GFX9-NEXT: s_branch .LBB47_6
+; GFX9-NEXT: .LBB47_5:
; GFX9-NEXT: v_mov_b32_e32 v13, s16
; GFX9-NEXT: v_mov_b32_e32 v14, s17
; GFX9-NEXT: v_mov_b32_e32 v8, s18
@@ -11137,7 +11667,7 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v10, s10
; GFX9-NEXT: v_mov_b32_e32 v11, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: .LBB47_5: ; %end
+; GFX9-NEXT: .LBB47_6: ; %end
; GFX9-NEXT: v_mov_b32_e32 v0, v13
; GFX9-NEXT: v_mov_b32_e32 v4, v14
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -11146,10 +11676,10 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB47_3
+; GFX11-NEXT: s_mov_b32 s8, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB47_2
; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s8, s2, 16
+; GFX11-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-NEXT: s_lshr_b32 s9, s2, 8
; GFX11-NEXT: s_lshr_b32 s10, s1, 24
; GFX11-NEXT: s_lshr_b32 s11, s1, 16
@@ -11158,9 +11688,25 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b32 s14, s0, 8
; GFX11-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s3
-; GFX11-NEXT: s_cbranch_vccnz .LBB47_4
-; GFX11-NEXT: .LBB47_2: ; %cmp.true
+; GFX11-NEXT: s_branch .LBB47_3
+; GFX11-NEXT: .LBB47_2:
+; GFX11-NEXT: s_mov_b32 s8, -1
+; GFX11-NEXT: ; implicit-def: $sgpr14
+; GFX11-NEXT: ; implicit-def: $sgpr12
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr13
+; GFX11-NEXT: ; implicit-def: $sgpr11
+; GFX11-NEXT: ; implicit-def: $sgpr10
+; GFX11-NEXT: ; implicit-def: $sgpr9
+; GFX11-NEXT: ; implicit-def: $sgpr3
+; GFX11-NEXT: ; implicit-def: $sgpr6
+; GFX11-NEXT: .LBB47_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB47_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v14, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v13, s0, 3 op_sel_hi:[1,0]
@@ -11174,27 +11720,16 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v13
-; GFX11-NEXT: s_branch .LBB47_5
-; GFX11-NEXT: .LBB47_3:
-; GFX11-NEXT: ; implicit-def: $sgpr14
-; GFX11-NEXT: ; implicit-def: $sgpr12
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr13
-; GFX11-NEXT: ; implicit-def: $sgpr11
-; GFX11-NEXT: ; implicit-def: $sgpr10
-; GFX11-NEXT: ; implicit-def: $sgpr9
-; GFX11-NEXT: ; implicit-def: $sgpr8
-; GFX11-NEXT: ; implicit-def: $sgpr6
-; GFX11-NEXT: s_branch .LBB47_2
-; GFX11-NEXT: .LBB47_4:
+; GFX11-NEXT: s_branch .LBB47_6
+; GFX11-NEXT: .LBB47_5:
; GFX11-NEXT: v_dual_mov_b32 v13, s0 :: v_dual_mov_b32 v14, s1
; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
; GFX11-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v5, s13
; GFX11-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
; GFX11-NEXT: v_mov_b32_e32 v11, s6
; GFX11-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-NEXT: .LBB47_5: ; %end
+; GFX11-NEXT: .LBB47_6: ; %end
; GFX11-NEXT: v_mov_b32_e32 v0, v13
; GFX11-NEXT: v_mov_b32_e32 v4, v14
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -11594,7 +12129,7 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s9
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; SI-NEXT: s_cbranch_scc0 .LBB49_4
+; SI-NEXT: s_cbranch_scc0 .LBB49_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
; SI-NEXT: v_lshr_b64 v[1:2], v[5:6], 16
@@ -11604,8 +12139,20 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v15
; SI-NEXT: v_lshr_b64 v[12:13], v[0:1], 16
; SI-NEXT: v_lshr_b64 v[8:9], v[3:4], 16
-; SI-NEXT: s_cbranch_execnz .LBB49_3
-; SI-NEXT: .LBB49_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB49_3
+; SI-NEXT: .LBB49_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr7
+; SI-NEXT: ; implicit-def: $vgpr12
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: .LBB49_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB49_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v1
@@ -11626,7 +12173,7 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v10
; SI-NEXT: v_lshr_b64 v[8:9], v[3:4], 16
; SI-NEXT: v_lshr_b64 v[12:13], v[0:1], 16
-; SI-NEXT: .LBB49_3: ; %end
+; SI-NEXT: .LBB49_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v7
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v12
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -11637,21 +12184,23 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB49_4:
-; SI-NEXT: ; implicit-def: $vgpr7
-; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: s_branch .LBB49_2
;
; VI-LABEL: bitcast_v6bf16_to_v6f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB49_3
+; VI-NEXT: s_cbranch_scc0 .LBB49_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB49_4
-; VI-NEXT: .LBB49_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB49_3
+; VI-NEXT: .LBB49_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB49_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB49_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v3, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v3
@@ -11709,9 +12258,7 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB49_3:
-; VI-NEXT: s_branch .LBB49_2
-; VI-NEXT: .LBB49_4:
+; VI-NEXT: .LBB49_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -11722,10 +12269,18 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB49_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB49_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB49_4
-; GFX9-NEXT: .LBB49_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB49_3
+; GFX9-NEXT: .LBB49_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB49_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB49_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -11786,9 +12341,7 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX9-NEXT: v_and_b32_sdwa v0, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB49_3:
-; GFX9-NEXT: s_branch .LBB49_2
-; GFX9-NEXT: .LBB49_4:
+; GFX9-NEXT: .LBB49_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -11800,11 +12353,16 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-TRUE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB49_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -11868,8 +12426,6 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB49_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB49_2
; GFX11-TRUE16-NEXT: .LBB49_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -11880,11 +12436,16 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB49_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB49_4
-; GFX11-FAKE16-NEXT: .LBB49_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB49_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB49_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -11952,8 +12513,6 @@ define inreg <6 x half> @bitcast_v6bf16_to_v6f16_scalar(<6 x bfloat> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v4, 16, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB49_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB49_2
; GFX11-FAKE16-NEXT: .LBB49_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12127,7 +12686,7 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s7, s17, 16
; SI-NEXT: s_lshr_b32 s6, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB51_3
+; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s9, s16, 16
; SI-NEXT: s_lshl_b32 s10, s6, 16
@@ -12135,8 +12694,22 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s12, s7, 16
; SI-NEXT: s_lshl_b32 s13, s18, 16
; SI-NEXT: s_lshl_b32 s14, s8, 16
-; SI-NEXT: s_cbranch_execnz .LBB51_4
-; SI-NEXT: .LBB51_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB51_3
+; SI-NEXT: .LBB51_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: ; implicit-def: $sgpr13
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: .LBB51_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB51_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s8
; SI-NEXT: v_cvt_f32_f16_e32 v1, s18
; SI-NEXT: v_cvt_f32_f16_e32 v2, s6
@@ -12161,23 +12734,15 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_branch .LBB51_5
-; SI-NEXT: .LBB51_3:
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: s_branch .LBB51_2
-; SI-NEXT: .LBB51_4:
+; SI-NEXT: s_branch .LBB51_6
+; SI-NEXT: .LBB51_5:
; SI-NEXT: v_mov_b32_e32 v4, s14
; SI-NEXT: v_mov_b32_e32 v3, s13
; SI-NEXT: v_mov_b32_e32 v5, s12
; SI-NEXT: v_mov_b32_e32 v2, s11
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v0, s9
-; SI-NEXT: .LBB51_5: ; %end
+; SI-NEXT: .LBB51_6: ; %end
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
@@ -12196,10 +12761,18 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB51_3
+; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB51_4
-; VI-NEXT: .LBB51_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB51_3
+; VI-NEXT: .LBB51_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB51_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB51_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -12217,9 +12790,7 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; VI-NEXT: v_or_b32_e32 v1, v1, v5
; VI-NEXT: v_or_b32_e32 v0, v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB51_3:
-; VI-NEXT: s_branch .LBB51_2
-; VI-NEXT: .LBB51_4:
+; VI-NEXT: .LBB51_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12230,18 +12801,24 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB51_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB51_4
-; GFX9-NEXT: .LBB51_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB51_3
+; GFX9-NEXT: .LBB51_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB51_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB51_3:
-; GFX9-NEXT: s_branch .LBB51_2
-; GFX9-NEXT: .LBB51_4:
+; GFX9-NEXT: .LBB51_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12253,17 +12830,20 @@ define inreg <6 x bfloat> @bitcast_v6f16_to_v6bf16_scalar(<6 x half> inreg %a, i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB51_4
-; GFX11-NEXT: .LBB51_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB51_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB51_3:
-; GFX11-NEXT: s_branch .LBB51_2
; GFX11-NEXT: .LBB51_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12657,7 +13237,7 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s4
-; SI-NEXT: s_cbranch_scc0 .LBB53_4
+; SI-NEXT: s_cbranch_scc0 .LBB53_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v11
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v4
@@ -12665,8 +13245,22 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v10
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v7
-; SI-NEXT: s_cbranch_execnz .LBB53_3
-; SI-NEXT: .LBB53_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB53_3
+; SI-NEXT: .LBB53_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr9
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: ; implicit-def: $vgpr5
+; SI-NEXT: .LBB53_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB53_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
@@ -12687,7 +13281,7 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
; SI-NEXT: v_lshr_b64 v[9:10], v[0:1], 16
-; SI-NEXT: .LBB53_3: ; %end
+; SI-NEXT: .LBB53_5: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v3
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v9
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -12698,23 +13292,23 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB53_4:
-; SI-NEXT: ; implicit-def: $vgpr3
-; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: s_branch .LBB53_2
;
; VI-LABEL: bitcast_v6bf16_to_v6i16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB53_3
+; VI-NEXT: s_cbranch_scc0 .LBB53_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB53_4
-; VI-NEXT: .LBB53_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB53_3
+; VI-NEXT: .LBB53_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB53_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB53_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v3, 0x40c00000
; VI-NEXT: v_add_f32_e32 v0, s4, v3
@@ -12772,9 +13366,7 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB53_3:
-; VI-NEXT: s_branch .LBB53_2
-; VI-NEXT: .LBB53_4:
+; VI-NEXT: .LBB53_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -12785,10 +13377,18 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB53_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB53_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB53_4
-; GFX9-NEXT: .LBB53_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB53_3
+; GFX9-NEXT: .LBB53_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB53_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB53_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: s_pack_lh_b32_b16 s4, 0, s16
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40c00000
; GFX9-NEXT: v_add_f32_e32 v1, s4, v0
@@ -12846,9 +13446,7 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v4
; GFX9-NEXT: v_and_or_b32 v0, v3, v6, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB53_3:
-; GFX9-NEXT: s_branch .LBB53_2
-; GFX9-NEXT: .LBB53_4:
+; GFX9-NEXT: .LBB53_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -12860,11 +13458,16 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-TRUE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: .LBB53_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -12919,8 +13522,6 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB53_3:
-; GFX11-TRUE16-NEXT: s_branch .LBB53_2
; GFX11-TRUE16-NEXT: .LBB53_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -12931,11 +13532,16 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB53_3
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-FAKE16-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: s_cbranch_vccnz .LBB53_4
-; GFX11-FAKE16-NEXT: .LBB53_2: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_2
+; GFX11-FAKE16-NEXT: ; %bb.1:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: .LBB53_2: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB53_4
+; GFX11-FAKE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
@@ -12994,8 +13600,6 @@ define inreg <6 x i16> @bitcast_v6bf16_to_v6i16_scalar(<6 x bfloat> inreg %a, i3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v0, v7
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB53_3:
-; GFX11-FAKE16-NEXT: s_branch .LBB53_2
; GFX11-FAKE16-NEXT: .LBB53_4:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13156,7 +13760,7 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s13, s17, 16
; SI-NEXT: s_lshr_b32 s12, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB55_4
+; SI-NEXT: s_cbranch_scc0 .LBB55_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_lshl_b32 s8, s16, 16
; SI-NEXT: s_lshl_b32 s11, s12, 16
@@ -13164,8 +13768,22 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s10, s13, 16
; SI-NEXT: s_lshl_b32 s6, s18, 16
; SI-NEXT: s_lshl_b32 s9, s14, 16
-; SI-NEXT: s_cbranch_execnz .LBB55_3
-; SI-NEXT: .LBB55_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB55_3
+; SI-NEXT: .LBB55_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: ; implicit-def: $sgpr8
+; SI-NEXT: ; implicit-def: $sgpr11
+; SI-NEXT: ; implicit-def: $sgpr7
+; SI-NEXT: ; implicit-def: $sgpr10
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr9
+; SI-NEXT: .LBB55_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB55_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s18, s18, 3
; SI-NEXT: s_and_b32 s4, s18, 0xffff
; SI-NEXT: s_lshl_b32 s5, s14, 16
@@ -13187,7 +13805,7 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; SI-NEXT: s_lshl_b32 s7, s5, 16
; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
; SI-NEXT: s_lshl_b32 s6, s4, 16
-; SI-NEXT: .LBB55_3: ; %end
+; SI-NEXT: .LBB55_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s11
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
@@ -13201,23 +13819,23 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s6
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB55_4:
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: ; implicit-def: $sgpr11
-; SI-NEXT: ; implicit-def: $sgpr7
-; SI-NEXT: ; implicit-def: $sgpr10
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr9
-; SI-NEXT: s_branch .LBB55_2
;
; VI-LABEL: bitcast_v6i16_to_v6bf16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB55_4
+; VI-NEXT: s_cbranch_scc0 .LBB55_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB55_3
-; VI-NEXT: .LBB55_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB55_3
+; VI-NEXT: .LBB55_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB55_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB55_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -13233,29 +13851,33 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB55_3: ; %end
+; VI-NEXT: .LBB55_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB55_4:
-; VI-NEXT: s_branch .LBB55_2
;
; GFX9-LABEL: bitcast_v6i16_to_v6bf16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB55_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB55_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB55_4
-; GFX9-NEXT: .LBB55_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB55_3
+; GFX9-NEXT: .LBB55_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB55_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB55_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB55_3:
-; GFX9-NEXT: s_branch .LBB55_2
-; GFX9-NEXT: .LBB55_4:
+; GFX9-NEXT: .LBB55_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13267,17 +13889,20 @@ define inreg <6 x bfloat> @bitcast_v6i16_to_v6bf16_scalar(<6 x i16> inreg %a, i3
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB55_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB55_4
-; GFX11-NEXT: .LBB55_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB55_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB55_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB55_3:
-; GFX11-NEXT: s_branch .LBB55_2
; GFX11-NEXT: .LBB55_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13429,10 +14054,18 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s6, s17, 16
; SI-NEXT: s_lshr_b32 s8, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB57_3
+; SI-NEXT: s_cbranch_scc0 .LBB57_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: s_cbranch_execnz .LBB57_4
-; SI-NEXT: .LBB57_2: ; %cmp.true
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_branch .LBB57_3
+; SI-NEXT: .LBB57_2:
+; SI-NEXT: s_mov_b64 s[4:5], -1
+; SI-NEXT: .LBB57_3: ; %Flow
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s4, 1
+; SI-NEXT: s_cbranch_scc1 .LBB57_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: v_cvt_f32_f16_e32 v0, s8
; SI-NEXT: v_cvt_f32_f16_e32 v1, s16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s7
@@ -13458,17 +14091,15 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_lshr_b64 v[2:3], v[0:1], 16
; SI-NEXT: v_or_b32_e32 v3, v8, v7
-; SI-NEXT: s_branch .LBB57_5
-; SI-NEXT: .LBB57_3:
-; SI-NEXT: s_branch .LBB57_2
-; SI-NEXT: .LBB57_4:
+; SI-NEXT: s_branch .LBB57_6
+; SI-NEXT: .LBB57_5:
; SI-NEXT: v_mov_b32_e32 v4, s7
; SI-NEXT: v_mov_b32_e32 v6, s6
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v3, s18
; SI-NEXT: v_mov_b32_e32 v5, s16
; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: .LBB57_5: ; %end
+; SI-NEXT: .LBB57_6: ; %end
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v5
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v0, v0, v2
@@ -13484,10 +14115,18 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB57_3
+; VI-NEXT: s_cbranch_scc0 .LBB57_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB57_4
-; VI-NEXT: .LBB57_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB57_3
+; VI-NEXT: .LBB57_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB57_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB57_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_lshr_b32 s4, s16, 16
; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_lshr_b32 s4, s17, 16
@@ -13505,9 +14144,7 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; VI-NEXT: v_or_b32_e32 v1, v1, v5
; VI-NEXT: v_or_b32_e32 v0, v3, v4
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB57_3:
-; VI-NEXT: s_branch .LBB57_2
-; VI-NEXT: .LBB57_4:
+; VI-NEXT: .LBB57_5:
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
@@ -13518,18 +14155,24 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB57_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB57_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB57_4
-; GFX9-NEXT: .LBB57_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB57_3
+; GFX9-NEXT: .LBB57_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB57_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_mov_b32_e32 v0, 0x200
; GFX9-NEXT: v_pk_add_f16 v2, s18, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v1, s17, v0 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_f16 v0, s16, v0 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB57_3:
-; GFX9-NEXT: s_branch .LBB57_2
-; GFX9-NEXT: .LBB57_4:
+; GFX9-NEXT: .LBB57_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13541,17 +14184,20 @@ define inreg <6 x i16> @bitcast_v6f16_to_v6i16_scalar(<6 x half> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB57_4
-; GFX11-NEXT: .LBB57_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB57_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_f16 v2, 0x200, s2 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v1, 0x200, s1 op_sel_hi:[0,1]
; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_3:
-; GFX11-NEXT: s_branch .LBB57_2
; GFX11-NEXT: .LBB57_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
@@ -13714,7 +14360,7 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; SI-NEXT: s_lshr_b32 s11, s17, 16
; SI-NEXT: s_lshr_b32 s13, s16, 16
; SI-NEXT: s_cmp_lg_u32 s19, 0
-; SI-NEXT: s_cbranch_scc0 .LBB59_4
+; SI-NEXT: s_cbranch_scc0 .LBB59_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s5, s18, 0xffff
; SI-NEXT: s_lshl_b32 s6, s10, 16
@@ -13726,9 +14372,20 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; SI-NEXT: s_or_b32 s15, s5, s6
; SI-NEXT: s_or_b32 s4, s4, s14
; SI-NEXT: s_lshr_b64 s[6:7], s[14:15], 16
+; SI-NEXT: s_mov_b64 s[8:9], 0
; SI-NEXT: s_mov_b32 s5, s15
-; SI-NEXT: s_cbranch_execnz .LBB59_3
-; SI-NEXT: .LBB59_2: ; %cmp.true
+; SI-NEXT: s_branch .LBB59_3
+; SI-NEXT: .LBB59_2:
+; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr12
+; SI-NEXT: .LBB59_3: ; %Flow
+; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, 1
+; SI-NEXT: s_cbranch_scc1 .LBB59_5
+; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s16, s16, 3
; SI-NEXT: s_and_b32 s4, s16, 0xffff
; SI-NEXT: s_lshl_b32 s5, s13, 16
@@ -13747,7 +14404,7 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; SI-NEXT: s_lshr_b64 s[6:7], s[4:5], 16
; SI-NEXT: s_lshr_b32 s11, s5, 16
; SI-NEXT: s_lshr_b32 s10, s12, 16
-; SI-NEXT: .LBB59_3: ; %end
+; SI-NEXT: .LBB59_5: ; %end
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -13761,20 +14418,23 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: s_setpc_b64 s[30:31]
-; SI-NEXT: .LBB59_4:
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr6
-; SI-NEXT: ; implicit-def: $sgpr12
-; SI-NEXT: s_branch .LBB59_2
;
; VI-LABEL: bitcast_v6i16_to_v6f16_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s19, 0
-; VI-NEXT: s_cbranch_scc0 .LBB59_4
+; VI-NEXT: s_cbranch_scc0 .LBB59_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: s_cbranch_execnz .LBB59_3
-; VI-NEXT: .LBB59_2: ; %cmp.true
+; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_branch .LBB59_3
+; VI-NEXT: .LBB59_2:
+; VI-NEXT: s_mov_b64 s[4:5], -1
+; VI-NEXT: .LBB59_3: ; %Flow
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_cmp_lg_u32 s4, 1
+; VI-NEXT: s_cbranch_scc1 .LBB59_5
+; VI-NEXT: ; %bb.4: ; %cmp.true
; VI-NEXT: s_add_i32 s5, s16, 3
; VI-NEXT: s_add_i32 s7, s17, 3
; VI-NEXT: s_add_i32 s9, s18, 3
@@ -13790,29 +14450,33 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; VI-NEXT: s_add_i32 s18, s8, 0x30000
; VI-NEXT: s_add_i32 s17, s6, 0x30000
; VI-NEXT: s_add_i32 s16, s4, 0x30000
-; VI-NEXT: .LBB59_3: ; %end
+; VI-NEXT: .LBB59_5: ; %end
; VI-NEXT: v_mov_b32_e32 v0, s16
; VI-NEXT: v_mov_b32_e32 v1, s17
; VI-NEXT: v_mov_b32_e32 v2, s18
; VI-NEXT: s_setpc_b64 s[30:31]
-; VI-NEXT: .LBB59_4:
-; VI-NEXT: s_branch .LBB59_2
;
; GFX9-LABEL: bitcast_v6i16_to_v6f16_scalar:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s19, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB59_3
+; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: s_cbranch_execnz .LBB59_4
-; GFX9-NEXT: .LBB59_2: ; %cmp.true
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_branch .LBB59_3
+; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_mov_b64 s[4:5], -1
+; GFX9-NEXT: .LBB59_3: ; %Flow
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX9-NEXT: ; %bb.4: ; %cmp.true
; GFX9-NEXT: v_pk_add_u16 v2, s18, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v1, s17, 3 op_sel_hi:[1,0]
; GFX9-NEXT: v_pk_add_u16 v0, s16, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
-; GFX9-NEXT: .LBB59_3:
-; GFX9-NEXT: s_branch .LBB59_2
-; GFX9-NEXT: .LBB59_4:
+; GFX9-NEXT: .LBB59_5:
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
@@ -13824,17 +14488,20 @@ define inreg <6 x half> @bitcast_v6i16_to_v6f16_scalar(<6 x i16> inreg %a, i32 i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_3
-; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_vccnz .LBB59_4
-; GFX11-NEXT: .LBB59_2: ; %cmp.true
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_2
+; GFX11-NEXT: ; %bb.1:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: .LBB59_2: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_4
+; GFX11-NEXT: ; %bb.3: ; %cmp.true
; GFX11-NEXT: v_pk_add_u16 v2, s2, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v1, s1, 3 op_sel_hi:[1,0]
; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB59_3:
-; GFX11-NEXT: s_branch .LBB59_2
; GFX11-NEXT: .LBB59_4:
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
index 92198f2593f3a..d35b3da59d92f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
@@ -138,9 +138,12 @@ define protected amdgpu_kernel void @trivial_waterfall_eq_zero(ptr addrspace(1)
; CHECK-NEXT: s_branch .LBB7_2
; CHECK-NEXT: .LBB7_1: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB7_2 Depth=1
-; CHECK-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
+; CHECK-NEXT: s_and_b32 s2, s2, exec_lo
+; CHECK-NEXT: s_cselect_b32 s2, 1, 0
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT: s_cmp_lg_u32 s2, 1
; CHECK-NEXT: s_mov_b32 s2, -1
-; CHECK-NEXT: s_cbranch_vccz .LBB7_4
+; CHECK-NEXT: s_cbranch_scc0 .LBB7_4
; CHECK-NEXT: .LBB7_2: ; %while
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s2
diff --git a/llvm/test/CodeGen/AMDGPU/anyext.ll b/llvm/test/CodeGen/AMDGPU/anyext.ll
index 1492119a6022d..835bee67376f1 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext.ll
@@ -15,8 +15,8 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u32 s6, 0
-; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT: s_cselect_b32 s4, 1, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
@@ -28,8 +28,8 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GFX8-NEXT: s_mov_b32 s2, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_cmp_lg_u32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -41,8 +41,8 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s6, 0
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
entry:
@@ -150,7 +150,9 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
; GCN-NEXT: v_and_b32_e32 v0, 0x8001, v0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_and_b64 s[0:1], vcc, exec
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
@@ -163,7 +165,9 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX8-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -177,7 +181,9 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
; GFX9-NEXT: v_and_b32_e32 v0, 0x80018001, v0
; GFX9-NEXT: v_bfi_b32 v0, v1, 0, v0
; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 15a43df327447..f0b994420a2ca 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -44,8 +44,10 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_wqm_b64 s[4:5], -1
; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX7-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT: s_cselect_b32 s4, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s4, 1
+; GFX7-NEXT: s_cbranch_scc1 .LBB0_6
; GFX7-NEXT: ; %bb.5: ; %if
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -79,8 +81,10 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX89-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX89-NEXT: s_wqm_b64 s[4:5], -1
; GFX89-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX89-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX89-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX89-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX89-NEXT: s_cselect_b32 s4, 1, 0
+; GFX89-NEXT: s_cmp_lg_u32 s4, 1
+; GFX89-NEXT: s_cbranch_scc1 .LBB0_6
; GFX89-NEXT: ; %bb.5: ; %if
; GFX89-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX89-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -115,8 +119,10 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_wqm_b64 s[4:5], -1
; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1064-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX1064-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1064-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1064-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1064-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1064-NEXT: ; %bb.5: ; %if
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1064-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -150,8 +156,10 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_wqm_b32 s4, -1
; GFX1032-NEXT: s_and_b32 s4, s4, s4
-; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
-; GFX1032-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1032-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1032-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1032-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1032-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1032-NEXT: ; %bb.5: ; %if
; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -190,8 +198,11 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1164-NEXT: s_wqm_b64 s[4:5], -1
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1164-NEXT: s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1164-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1164-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1164-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1164-NEXT: ; %bb.5: ; %if
; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1164-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -229,8 +240,11 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1132-NEXT: s_wqm_b32 s4, -1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_and_b32 s4, s4, s4
-; GFX1132-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1132-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1132-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1132-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1132-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1132-NEXT: ; %bb.5: ; %if
; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1132-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -269,8 +283,11 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1364-NEXT: s_wqm_b64 s[4:5], -1
; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1364-NEXT: s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1364-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1364-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1364-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1364-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1364-NEXT: ; %bb.5: ; %if
; GFX1364-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1364-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -308,8 +325,11 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1332-NEXT: s_wqm_b32 s4, -1
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_and_b32 s4, s4, s4
-; GFX1332-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1332-NEXT: s_cbranch_vccnz .LBB0_6
+; GFX1332-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1332-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1332-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1332-NEXT: ; %bb.5: ; %if
; GFX1332-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1332-NEXT: .LBB0_6: ; %UnifiedReturnBlock
@@ -333,8 +353,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: buffer_atomic_add v0, off, s[4:7], 0 glc
; GFX7-NEXT: s_wqm_b64 s[4:5], -1
-; GFX7-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_vccnz .LBB1_2
+; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT: s_cselect_b32 s4, 1, 0
+; GFX7-NEXT: s_cmp_lg_u32 s4, 1
+; GFX7-NEXT: s_cbranch_scc1 .LBB1_2
; GFX7-NEXT: ; %bb.1: ; %if
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -389,8 +411,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_wqm_b64 s[4:5], -1
; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX8-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX8-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: s_cmp_lg_u32 s4, 1
+; GFX8-NEXT: s_cbranch_scc1 .LBB1_6
; GFX8-NEXT: ; %bb.5: ; %if
; GFX8-NEXT: buffer_store_dword v3, off, s[0:3], 0
; GFX8-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -444,8 +468,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_wqm_b64 s[4:5], -1
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: s_cmp_lg_u32 s4, 1
+; GFX9-NEXT: s_cbranch_scc1 .LBB1_6
; GFX9-NEXT: ; %bb.5: ; %if
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], 0
; GFX9-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -504,8 +530,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_wqm_b64 s[4:5], -1
; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1064-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GFX1064-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1064-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1064-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1064-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1064-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1064-NEXT: ; %bb.5: ; %if
; GFX1064-NEXT: buffer_store_dword v4, off, s[0:3], 0
; GFX1064-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -554,8 +582,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_wqm_b32 s4, -1
; GFX1032-NEXT: s_and_b32 s4, s4, s4
-; GFX1032-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
-; GFX1032-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1032-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1032-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1032-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1032-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1032-NEXT: ; %bb.5: ; %if
; GFX1032-NEXT: buffer_store_dword v4, off, s[0:3], 0
; GFX1032-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -625,8 +655,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1164-NEXT: s_wqm_b64 s[4:5], -1
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1164-NEXT: s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1164-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1164-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1164-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1164-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1164-NEXT: ; %bb.5: ; %if
; GFX1164-NEXT: buffer_store_b32 v4, off, s[0:3], 0
; GFX1164-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -682,8 +715,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1132-NEXT: s_wqm_b32 s4, -1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_and_b32 s4, s4, s4
-; GFX1132-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1132-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1132-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1132-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1132-NEXT: ; %bb.5: ; %if
; GFX1132-NEXT: buffer_store_b32 v4, off, s[0:3], 0
; GFX1132-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -752,8 +788,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1364-NEXT: s_wqm_b64 s[4:5], -1
; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: s_and_b64 s[4:5], s[4:5], s[4:5]
-; GFX1364-NEXT: s_and_not1_b64 vcc, exec, s[4:5]
-; GFX1364-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1364-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX1364-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1364-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1364-NEXT: ; %bb.5: ; %if
; GFX1364-NEXT: buffer_store_b32 v4, off, s[0:3], null
; GFX1364-NEXT: .LBB1_6: ; %UnifiedReturnBlock
@@ -808,8 +847,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1332-NEXT: s_wqm_b32 s4, -1
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_and_b32 s4, s4, s4
-; GFX1332-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX1332-NEXT: s_cbranch_vccnz .LBB1_6
+; GFX1332-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1332-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: s_cmp_lg_u32 s4, 1
+; GFX1332-NEXT: s_cbranch_scc1 .LBB1_6
; GFX1332-NEXT: ; %bb.5: ; %if
; GFX1332-NEXT: buffer_store_b32 v4, off, s[0:3], null
; GFX1332-NEXT: .LBB1_6: ; %UnifiedReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 8d2e1c8b89ffd..9c379996385ce 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -5,20 +5,17 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-LABEL: vgpr_mfma_pass_av_split_crash:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x0
-; CHECK-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x10
+; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[16:17], -1, 0
-; CHECK-NEXT: s_xor_b64 s[18:19], s[16:17], -1
+; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
+; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; CHECK-NEXT: s_xor_b64 s[20:21], s[2:3], -1
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
-; CHECK-NEXT: s_and_b64 s[2:3], exec, s[2:3]
+; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
+; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
; CHECK-NEXT: v_mov_b32_e32 v4, 0x19f4ec90
@@ -31,15 +28,16 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v11, 0xbf8c6ea4
; CHECK-NEXT: v_mov_b32_e32 v12, 0xe82d3ff0
; CHECK-NEXT: v_mov_b32_e32 v13, 0xbfa59976
+; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
-; CHECK-NEXT: s_mov_b64 s[22:23], 0
+; CHECK-NEXT: s_and_b64 s[0:1], exec, s[14:15]
; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
-; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
+; CHECK-NEXT: s_and_b64 s[2:3], exec, s[18:19]
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
-; CHECK-NEXT: s_and_b64 s[6:7], exec, s[18:19]
+; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
@@ -47,20 +45,24 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_branch .LBB0_2
; CHECK-NEXT: .LBB0_1: ; %Flow9
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_andn2_b64 vcc, exec, s[24:25]
-; CHECK-NEXT: s_cbranch_vccz .LBB0_17
+; CHECK-NEXT: s_and_b64 s[6:7], s[24:25], exec
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s6, 1
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_17
; CHECK-NEXT: .LBB0_2: ; %._crit_edge1942.i.i.i3548
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB0_6 Depth 2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[0:1]
-; CHECK-NEXT: s_cbranch_vccnz .LBB0_9
+; CHECK-NEXT: s_and_b64 s[6:7], exec, s[18:19]
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s6, 1
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_9
; CHECK-NEXT: ; %bb.3: ; %.preheader1868.i.i.i3244
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 vcc, s[4:5]
+; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
; CHECK-NEXT: s_cbranch_vccz .LBB0_10
; CHECK-NEXT: ; %bb.4: ; %.preheader1855.i.i.i3329.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
@@ -88,7 +90,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_branch .LBB0_6
; CHECK-NEXT: .LBB0_5: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[8:9]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_11
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
@@ -96,7 +98,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
; CHECK-NEXT: s_mov_b64 s[24:25], -1
-; CHECK-NEXT: s_mov_b64 s[8:9], -1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
; CHECK-NEXT: ; implicit-def: $agpr2_agpr3
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
@@ -104,25 +106,25 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
; CHECK-NEXT: v_accvgpr_mov_b32 a2, a0
-; CHECK-NEXT: s_mov_b64 s[8:9], s[18:19]
-; CHECK-NEXT: s_mov_b64 vcc, s[6:7]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[16:17]
+; CHECK-NEXT: s_mov_b64 vcc, s[4:5]
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
; CHECK-NEXT: s_mov_b64 s[24:25], 0
; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[12:13]
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
-; CHECK-NEXT: s_mov_b64 s[8:9], s[20:21]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[20:21]
; CHECK-NEXT: s_branch .LBB0_15
; CHECK-NEXT: .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[8:9], -1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
; CHECK-NEXT: s_branch .LBB0_15
; CHECK-NEXT: .LBB0_11: ; %loop.exit.guard
@@ -131,21 +133,21 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_vccz .LBB0_13
; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[8:9], 0, v[26:27]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[16:17]
-; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[8:9]
+; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
; CHECK-NEXT: v_mov_b32_e32 v27, v26
-; CHECK-NEXT: s_and_b64 s[8:9], exec, s[16:17]
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[16:17]
-; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[12:13]
+; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
+; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[8:9]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
-; CHECK-NEXT: s_mov_b64 s[8:9], -1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
; CHECK-NEXT: .LBB0_14: ; %Flow6
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
@@ -153,12 +155,12 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: .LBB0_15: ; %Flow6
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], -1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[8:9]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccz .LBB0_1
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[12:13]
+; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
index 554d7eb2cfbdb..dda8f1e343a2c 100644
--- a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
@@ -40,12 +40,14 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float>